cli: add client package, statestore implementations, deprecate config v1

GitOrigin-RevId: 5a1d9cb

.circleci/test-cli-with-last-release.sh

@@ -33,7 +33,7 @@ PID=$!
 wait_for_port 8080
 
 # test cli
-HASURA_GRAPHQL_TEST_ENDPOINT="http://localhost:8080" HASURA_GRAPHQL_TEST_ADMIN_SECRET="abcd" HASURA_GRAPHQL_TEST_CLI_EXT_MANIFEST_FILE_PATH="/build/_cli_ext_output/manifest-dev.yaml" SKIP_METADATA_TESTS="true" make test
+HASURA_GRAPHQL_TEST_ENDPOINT="http://localhost:8080" HASURA_GRAPHQL_TEST_ADMIN_SECRET="abcd" HASURA_GRAPHQL_TEST_CLI_EXT_MANIFEST_FILE_PATH="/build/_cli_ext_output/manifest-dev.yaml" make integration_tests_config_v2
 
 # kill the running server
 kill -s INT $PID
@@ -47,5 +47,6 @@ PID=$!
 wait_for_port 8080
 
 # test cli
-HASURA_GRAPHQL_TEST_ENDPOINT="http://localhost:8080" HASURA_GRAPHQL_TEST_ADMIN_SECRET="abcd" HASURA_GRAPHQL_TEST_CLI_EXT_MANIFEST_FILE_PATH="/build/_cli_ext_output/manifest-dev.yaml" SKIP_METADATA_TESTS="true" make test
+HASURA_GRAPHQL_TEST_ENDPOINT="http://localhost:8080" HASURA_GRAPHQL_TEST_ADMIN_SECRET="abcd" HASURA_GRAPHQL_TEST_CLI_EXT_MANIFEST_FILE_PATH="/build/_cli_ext_output/manifest-dev.yaml" make integration_tests_config_v2
+
 kill -s INT $PID

.circleci/test-cli.sh

@@ -29,7 +29,7 @@ PID=$!
 wait_for_port 8080
 
 # test cli
-HASURA_GRAPHQL_TEST_ENDPOINT="http://localhost:8080" 	HASURA_GRAPHQL_TEST_CLI_EXT_MANIFEST_FILE_PATH="/build/_cli_ext_output/manifest-dev.yaml" make test
+HASURA_GRAPHQL_TEST_ENDPOINT="http://localhost:8080" 	HASURA_GRAPHQL_TEST_CLI_EXT_MANIFEST_FILE_PATH="/build/_cli_ext_output/manifest-dev.yaml" make test-all
 kill -s INT $PID
 
 # start graphql-engine with admin secret
@@ -41,5 +41,5 @@ PID=$!
 wait_for_port 8080
 
 # test cli
-HASURA_GRAPHQL_TEST_ENDPOINT="http://localhost:8080" HASURA_GRAPHQL_TEST_CLI_EXT_MANIFEST_FILE_PATH="/build/_cli_ext_output/manifest-dev.yaml" HASURA_GRAPHQL_TEST_ADMIN_SECRET="abcd" make test
+HASURA_GRAPHQL_TEST_ENDPOINT="http://localhost:8080" HASURA_GRAPHQL_TEST_CLI_EXT_MANIFEST_FILE_PATH="/build/_cli_ext_output/manifest-dev.yaml" HASURA_GRAPHQL_TEST_ADMIN_SECRET="abcd" make test-all
 kill -s INT $PID

cli/Makefile

@@ -50,10 +50,11 @@ ci-copy-binary:
 .PHONY: test
 test:
 	go test -ldflags "-X github.com/hasura/graphql-engine/cli/version.BuildVersion=$(VERSION)" -v -tags="${TEST_TAGS}" `go list ./... | grep -v integration_test`
-	go test -ldflags "-X github.com/hasura/graphql-engine/cli/version.BuildVersion=$(VERSION)" -v -tags="${TEST_TAGS}" -run Commands/config=v1 ./integration_test
-	go test -ldflags "-X github.com/hasura/graphql-engine/cli/version.BuildVersion=$(VERSION)" -v -tags="${TEST_TAGS}" -run Commands/config=v2 ./integration_test
+integration_tests_config_v3:
 	go test -ldflags "-X github.com/hasura/graphql-engine/cli/version.BuildVersion=$(VERSION)" -v -tags="${TEST_TAGS}" -run Commands/config=v3 ./integration_test
-
+integration_tests_config_v2:
+	go test -ldflags "-X github.com/hasura/graphql-engine/cli/version.BuildVersion=$(VERSION)" -v -tags="${TEST_TAGS}" -run Commands/config=v2 ./integration_test
+test-all: test integration_tests_config_v2 integration_tests_config_v3
 # clean the output directory
 .PHONY: clean
 clean:

cli/cli.go

@@ -23,7 +23,21 @@ import (
 	"strings"
 	"time"
 
-	"github.com/hasura/graphql-engine/cli/internal/client"
+	"github.com/hasura/graphql-engine/cli/internal/hasura/v1metadata"
+	"github.com/hasura/graphql-engine/cli/internal/hasura/v1query"
+	"github.com/hasura/graphql-engine/cli/internal/hasura/v2query"
+
+	"github.com/hasura/graphql-engine/cli/internal/hasura/commonmetadata"
+
+	"github.com/hasura/graphql-engine/cli/internal/httpc"
+
+	"github.com/hasura/graphql-engine/cli/internal/statestore/settings"
+
+	"github.com/hasura/graphql-engine/cli/internal/statestore/migrations"
+
+	"github.com/hasura/graphql-engine/cli/internal/statestore"
+
+	"github.com/hasura/graphql-engine/cli/internal/hasura"
 
 	"github.com/Masterminds/semver"
 	"github.com/briandowns/spinner"
@@ -65,8 +79,6 @@ const (
 	XHasuraAccessKey   = "X-Hasura-Access-Key"
 )
 
-
-
 // String constants
 const (
 	StrTelemetryNotice = `Help us improve Hasura! The cli collects anonymized usage stats which
@@ -88,30 +100,13 @@ const (
 
 // ServerAPIPaths has the custom paths defined for server api
 type ServerAPIPaths struct {
-	Query string `yaml:"query,omitempty"`
-	// Metadata relevant since v1.4.0-alpha.1
-	// since v1/query is now split into v1/metadata and v2/query to support multiple datasources
-	Metadata string `yaml:"metadata,omitempty"`
-	GraphQL  string `yaml:"graphql,omitempty"`
-	Config   string `yaml:"config,omitempty"`
-	PGDump   string `yaml:"pg_dump,omitempty"`
-	Version  string `yaml:"version,omitempty"`
-}
-
-// SetDefaults will set default config values for API Paths
-// This function depends on server feature flags and will panic
-func (s *ServerAPIPaths) SetDefaults(hasMetadataV3 bool, configVersion ConfigVersion) {
-	if s.Query == "" {
-		if hasMetadataV3 && configVersion > V1 {
-			s.Query = "v2/query"
-		} else {
-			s.Query = "v1/query"
-		}
-	}
-
-	if s.Metadata == "" {
-		s.Metadata = "v1/metadata"
-	}
+	V1Query    string `yaml:"v1_query,omitempty"`
+	V2Query    string `yaml:"v2_query,omitempty"`
+	V1Metadata string `yaml:"v1_metadata,omitempty"`
+	GraphQL    string `yaml:"graphql,omitempty"`
+	Config     string `yaml:"config,omitempty"`
+	PGDump     string `yaml:"pg_dump,omitempty"`
+	Version    string `yaml:"version,omitempty"`
 }
 
 // GetQueryParams - encodes the values in url
@@ -245,16 +240,22 @@ func (s *ServerConfig) GetVersionEndpoint() string {
 }
 
 // GetQueryEndpoint provides the url to contact the query API
-func (s *ServerConfig) GetQueryEndpoint() string {
+func (s *ServerConfig) GetV1QueryEndpoint() string {
 	nurl := *s.ParsedEndpoint
-	nurl.Path = path.Join(nurl.Path, s.APIPaths.Query)
+	nurl.Path = path.Join(nurl.Path, s.APIPaths.V1Query)
+	return nurl.String()
+}
+
+func (s *ServerConfig) GetV2QueryEndpoint() string {
+	nurl := *s.ParsedEndpoint
+	nurl.Path = path.Join(nurl.Path, s.APIPaths.V2Query)
 	return nurl.String()
 }
 
 // GetQueryEndpoint provides the url to contact the query API
-func (s *ServerConfig) GetMetadataEndpoint() string {
+func (s *ServerConfig) GetV1MetadataEndpoint() string {
 	nurl := *s.ParsedEndpoint
-	nurl.Path = path.Join(nurl.Path, s.APIPaths.Metadata)
+	nurl.Path = path.Join(nurl.Path, s.APIPaths.V1Metadata)
 	return nurl.String()
 }
 
@@ -424,7 +425,7 @@ type ExecutionContext struct {
 	IsTerminal bool
 
 	// instance of API client which communicates with Hasura API
-	APIClient *client.Client
+	APIClient *hasura.Client
 
 	// current datasource on which operation is being done
 	Datasource    string
@@ -635,62 +636,55 @@ func (ec *ExecutionContext) Validate() error {
 	if err != nil {
 		return errors.Wrap(err, "error in getting server feature flags")
 	}
+	var headers map[string]string
 	if ec.Config.AdminSecret != "" {
-		headers := map[string]string{
+		headers = map[string]string{
 			GetAdminSecretHeaderName(ec.Version): ec.Config.AdminSecret,
 		}
 		ec.SetHGEHeaders(headers)
 	}
 
+	if !strings.HasSuffix(ec.Config.Endpoint, "/") {
+		ec.Config.Endpoint = fmt.Sprintf("%s/", ec.Config.Endpoint)
+	}
+	httpClient, err := httpc.New(nil, ec.Config.Endpoint, headers)
+	if err != nil {
+		return err
+	}
 	// check if server is using metadata v3
 	requestUri := ""
-	if ec.Config.GetMetadataEndpoint() != ec.Config.Endpoint {
-		requestUri = ec.Config.GetMetadataEndpoint()
-	}else {
-		requestUri = fmt.Sprintf("%s/%s", ec.Config.Endpoint, "v1/metadata")
+	if ec.Config.APIPaths.V1Query != "" {
+		requestUri = fmt.Sprintf("%s/%s", ec.Config.Endpoint, ec.Config.APIPaths.V1Query)
+	} else {
+		requestUri = fmt.Sprintf("%s/%s", ec.Config.Endpoint, "v1/query")
 	}
-	req, err := http.NewRequest(http.MethodPost, requestUri, strings.NewReader(`{"type": "export_metadata", "args": {}}`));
+	metadata, err := commonmetadata.New(httpClient, requestUri).ExportMetadata()
 	if err != nil {
 		return err
 	}
-	apiHttpClient := &http.Client{
-		Timeout:       time.Minute * 2,
+	var v struct {
+		Version int `json:"version"`
 	}
-	if ec.Config.TLSConfig != nil {
-		apiHttpClient.Transport = &http.Transport{TLSClientConfig: ec.Config.TLSConfig}
+	if err := json.NewDecoder(metadata).Decode(&v); err != nil {
+		return err
 	}
-	for k,v := range ec.HGEHeaders {
-		req.Header.Set(k,v)
+	if v.Version == 3 {
+		ec.HasMetadataV3 = true
 	}
-	resp, err  := apiHttpClient.Do(req)
-	if resp.StatusCode == http.StatusOK {
-		body,err := ioutil.ReadAll(resp.Body)
-		if err != nil {
-			return err
-		}
-		var v struct {
-			Version int `json:"version"`
-		}
-		if err := json.Unmarshal(body, &v); err != nil {
-			return err
-		}
-		if v.Version == 3 {
-			ec.HasMetadataV3 = true
-		}
-	}else if err != nil {
-		ec.Logger.Warn("checking if server supports metadata v3: ",err)
+	if ec.Config.Version >= V3 && !ec.HasMetadataV3 {
+		return fmt.Errorf("config v3 can only be used with servers having metadata version >= 3")
 	}
 
-	// set default API Paths w.r.t to server feature flags
-	ec.Config.ServerConfig.APIPaths.SetDefaults(ec.HasMetadataV3, ec.Config.Version)
-
-	ec.APIClient = client.NewClient(ec.HasMetadataV3)
-
+	ec.APIClient = &hasura.Client{
+		V1Metadata: v1metadata.New(httpClient, ec.Config.GetV1MetadataEndpoint()),
+		V1Query:    v1query.New(httpClient, ec.Config.GetV1QueryEndpoint()),
+		V2Query:    v2query.New(httpClient, ec.Config.GetV2QueryEndpoint()),
+	}
 	var state *util.ServerState
 	if ec.HasMetadataV3 {
-		state = util.GetServerState(ec.Config.GetMetadataEndpoint(), ec.Config.ServerConfig.AdminSecret, ec.Config.ServerConfig.TLSConfig, ec.HasMetadataV3, ec.Logger)
-	}else {
-		state = util.GetServerState(ec.Config.GetQueryEndpoint(), ec.Config.ServerConfig.AdminSecret, ec.Config.ServerConfig.TLSConfig, ec.HasMetadataV3, ec.Logger)
+		state = util.GetServerState(ec.Config.GetV1MetadataEndpoint(), ec.Config.ServerConfig.AdminSecret, ec.Config.ServerConfig.TLSConfig, ec.HasMetadataV3, ec.Logger)
+	} else {
+		state = util.GetServerState(ec.Config.GetV1QueryEndpoint(), ec.Config.ServerConfig.AdminSecret, ec.Config.ServerConfig.TLSConfig, ec.HasMetadataV3, ec.Logger)
 	}
 	ec.ServerUUID = state.UUID
 	ec.Telemetry.ServerUUID = ec.ServerUUID
@@ -745,8 +739,9 @@ func (ec *ExecutionContext) readConfig() error {
 	v.SetDefault("endpoint", "http://localhost:8080")
 	v.SetDefault("admin_secret", "")
 	v.SetDefault("access_key", "")
-	v.SetDefault("api_paths.query", "")
-	v.SetDefault("api_paths.metadata", "")
+	v.SetDefault("api_paths.query", "v1/query")
+	v.SetDefault("api_paths.v2_query", "v2/query")
+	v.SetDefault("api_paths.v1_metadata", "v1/metadata")
 	v.SetDefault("api_paths.graphql", "v1/graphql")
 	v.SetDefault("api_paths.config", "v1alpha1/config")
 	v.SetDefault("api_paths.pg_dump", "v1alpha1/pg_dump")
@@ -775,11 +770,13 @@ func (ec *ExecutionContext) readConfig() error {
 			Endpoint:    v.GetString("endpoint"),
 			AdminSecret: adminSecret,
 			APIPaths: &ServerAPIPaths{
-				Query:   v.GetString("api_paths.query"),
-				GraphQL: v.GetString("api_paths.graphql"),
-				Config:  v.GetString("api_paths.config"),
-				PGDump:  v.GetString("api_paths.pg_dump"),
-				Version: v.GetString("api_paths.version"),
+				V1Query:    v.GetString("api_paths.query"),
+				V2Query:    v.GetString("api_paths.v2_query"),
+				V1Metadata: v.GetString("api_paths.v1_metadata"),
+				GraphQL:    v.GetString("api_paths.graphql"),
+				Config:     v.GetString("api_paths.config"),
+				PGDump:     v.GetString("api_paths.pg_dump"),
+				Version:    v.GetString("api_paths.version"),
 			},
 			InsecureSkipTLSVerify: v.GetBool("insecure_skip_tls_verify"),
 			CAPath:                v.GetString("certificate_authority"),
@@ -938,3 +935,46 @@ func GetAdminSecretHeaderName(v *version.Version) string {
 	}
 	return XHasuraAdminSecret
 }
+func GetDatasourceOps(ec *ExecutionContext) hasura.DatasourceOperations {
+	if !ec.HasMetadataV3 {
+		return ec.APIClient.V1Query
+	}
+	return ec.APIClient.V2Query
+}
+
+func GetCommonMetadataOps(ec *ExecutionContext) hasura.CommonMetadataOperations {
+	if !ec.HasMetadataV3 {
+		return ec.APIClient.V1Query
+	}
+	return ec.APIClient.V1Metadata
+}
+
+func GetMigrationsStateStore(ec *ExecutionContext) statestore.MigrationsStateStore {
+	const (
+		defaultMigrationsTable = "schema_migrations"
+		defaultSchema          = "hdb_catalog"
+	)
+
+	if ec.Config.Version <= V2 {
+		if !ec.HasMetadataV3 {
+			return migrations.NewMigrationStateStoreHdbTable(ec.APIClient.V1Query, defaultSchema, defaultMigrationsTable)
+		}
+		return migrations.NewMigrationStateStoreHdbTable(ec.APIClient.V2Query, defaultSchema, defaultMigrationsTable)
+	}
+	return migrations.NewCatalogStateStore(statestore.NewCLICatalogState(ec.APIClient.V1Metadata))
+}
+
+func GetSettingsStateStore(ec *ExecutionContext) statestore.SettingsStateStore {
+	const (
+		defaultSettingsTable = "migration_settings"
+		defaultSchema        = "hdb_catalog"
+	)
+
+	if ec.Config.Version <= V2 {
+		if !ec.HasMetadataV3 {
+			return settings.NewStateStoreHdbTable(ec.APIClient.V1Query, defaultSchema, defaultSettingsTable)
+		}
+		return settings.NewStateStoreHdbTable(ec.APIClient.V2Query, defaultSchema, defaultSettingsTable)
+	}
+	return settings.NewStateStoreCatalog(statestore.NewCLICatalogState(ec.APIClient.V1Metadata))
+}

cli/commands/console.go

@@ -1,10 +1,10 @@
 package commands
 
 import (
-	"fmt"
 	"os"
 	"sync"
 
+	"github.com/hasura/graphql-engine/cli/internal/scripts"
 	"github.com/hasura/graphql-engine/cli/util"
 
 	"github.com/gin-gonic/gin"
@@ -49,12 +49,7 @@ func NewConsoleCmd(ec *cli.ExecutionContext) *cobra.Command {
 			if err := ec.Validate(); err != nil {
 				return err
 			}
-			if ec.Config.Version < cli.V2 && ec.HasMetadataV3 {
-				ec.Logger.Errorf("config V1 is not supported with metadata V3 please upgrade to config V2")
-				ec.Logger.Info("upgrade to config V2 using the following command\nhasura scripts update-project-v2")
-				return fmt.Errorf("invalid config version")
-			}
-			return nil
+			return scripts.CheckIfUpdateToConfigV3IsRequired(ec)
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return opts.Run()

cli/commands/metadata.go

@@ -2,6 +2,7 @@ package commands
 
 import (
 	"github.com/hasura/graphql-engine/cli"
+	"github.com/hasura/graphql-engine/cli/internal/scripts"
 	"github.com/hasura/graphql-engine/cli/migrate"
 	"github.com/hasura/graphql-engine/cli/util"
 	"github.com/pkg/errors"
@@ -23,7 +24,11 @@ func NewMetadataCmd(ec *cli.ExecutionContext) *cobra.Command {
 			if err != nil {
 				return err
 			}
-			return ec.Validate()
+			err = ec.Validate()
+			if err != nil {
+				return err
+			}
+			return scripts.CheckIfUpdateToConfigV3IsRequired(ec)
 		},
 		SilenceUsage: true,
 	}

cli/commands/migrate.go

@@ -3,6 +3,8 @@ package commands
 import (
 	"fmt"
 
+	"github.com/hasura/graphql-engine/cli/internal/scripts"
+
 	"github.com/hasura/graphql-engine/cli/util"
 	"github.com/pkg/errors"
 
@@ -39,7 +41,7 @@ func NewMigrateCmd(ec *cli.ExecutionContext) *cobra.Command {
 					return errors.New("datasource flag is required")
 				}
 			} else {
-				if err := checkIfUpdateToConfigV3IsRequired(ec); err != nil {
+				if err := scripts.CheckIfUpdateToConfigV3IsRequired(ec); err != nil {
 					return err
 				}
 			}

cli/commands/migrate_create.go

@@ -151,7 +151,7 @@ func (o *migrateCreateOptions) run() (version int64, err error) {
 		}
 	}
 	if o.sqlServer {
-		data, err := migrateDrv.ExportSchemaDump(o.schemaNames)
+		data, err := migrateDrv.ExportSchemaDump(o.schemaNames, o.Datasource)
 		if err != nil {
 			return 0, errors.Wrap(err, "cannot fetch schema dump")
 		}