feat: support sharding migration (#555)

* feat: support sharding migration

* chore: update golang version

* chore: update feature version of ShardingMigration

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21 as builder
+FROM golang:1.23 as builder
 
 ARG GOPROXY="https://proxy.golang.org,direct"
 

api/core/v1alpha1/cnclaim_types.go

@@ -106,6 +106,8 @@ type MigrateStatus struct {
 type Workload struct {
 	Connections int `json:"connections,omitempty"`
 	Pipelines   int `json:"pipelines,omitempty"`
+	// Replicas is the sum of sharding tables served on the current CN
+	Replicas int `json:"replicas,omitempty"`
 }
 
 // +kubebuilder:object:root=true

api/core/v1alpha1/semver.go

@@ -23,15 +23,17 @@ const (
 	MOFeatureSessionSource MOFeature = "SessionSource"
 	MOFeatureLockMigration MOFeature = "LockMigration"
 
-	MOFeatureDiscoveryFixed MOFeature = "DiscoveryFixed"
+	MOFeatureDiscoveryFixed    MOFeature = "DiscoveryFixed"
+	MOFeatureShardingMigration MOFeature = "ShardingMigration"
 )
 
 var (
 	featureVersions = map[MOFeature][]semver.Version{
-		MOFeaturePipelineInfo:   {semver.MustParse("1.1.2"), semver.MustParse("1.2.0")},
-		MOFeatureSessionSource:  {semver.MustParse("1.1.2"), semver.MustParse("1.2.0")},
-		MOFeatureLockMigration:  {semver.MustParse("1.2.0")},
-		MOFeatureDiscoveryFixed: {semver.MustParse("2.0.0")},
+		MOFeaturePipelineInfo:      {semver.MustParse("1.1.2"), semver.MustParse("1.2.0")},
+		MOFeatureSessionSource:     {semver.MustParse("1.1.2"), semver.MustParse("1.2.0")},
+		MOFeatureLockMigration:     {semver.MustParse("1.2.0")},
+		MOFeatureShardingMigration: {semver.MustParse("2.0.0")},
+		MOFeatureDiscoveryFixed:    {semver.MustParse("2.0.0")},
 	}
 
 	MinimalVersion = semver.Version{Major: 0, Minor: 0, Patch: 0}

charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnclaims.yaml

@@ -160,6 +160,10 @@ spec:
                         type: integer
                       pipelines:
                         type: integer
+                      replicas:
+                        description: Replicas is the sum of sharding tables served
+                          on the current CN
+                        type: integer
                     type: object
                 type: object
               phase:

charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnclaimsets.yaml

@@ -232,6 +232,10 @@ spec:
                               type: integer
                             pipelines:
                               type: integer
+                            replicas:
+                              description: Replicas is the sum of sharding tables
+                                served on the current CN
+                              type: integer
                           type: object
                       type: object
                     phase:

cmd/hacli/main.go

@@ -31,7 +31,7 @@ func main() {
 	zapLogger := logzap.NewRaw()
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	cli, err := logservice.NewProxyHAKeeperClient(ctx, logservice.HAKeeperClientConfig{ServiceAddresses: []string{"127.0.0.1:32001"}})
+	cli, err := logservice.NewProxyHAKeeperClient(ctx, "", logservice.HAKeeperClientConfig{ServiceAddresses: []string{"127.0.0.1:32001"}})
 	if err != nil {
 		panic(err)
 	}

cmd/operator/main.go

@@ -241,7 +241,7 @@ func main() {
 		setupLog.Info(fmt.Sprintf("s3 reclaim feature not enabled, skip setup bucketclaim actor"))
 	}
 
-	qc, err := querycli.New(zapLogger.Named("querycli"))
+	qc, err := querycli.New()
 	exitIf(err, "unable to create query client")
 	haCliMgr := mocli.NewManager(mgr.GetClient(), zapLogger.Named("mocli-manager"))
 	if features.DefaultFeatureGate.Enabled(features.CNLabel) {

cmd/rpccli/main.go

@@ -24,7 +24,7 @@ import (
 
 func main() {
 	zapLogger := logzap.NewRaw()
-	qc, err := querycli.New(zapLogger.Named("querycli"))
+	qc, err := querycli.New()
 	if err != nil {
 		panic(err)
 	}

deploy/crds/core.matrixorigin.io_cnclaims.yaml

@@ -160,6 +160,10 @@ spec:
                         type: integer
                       pipelines:
                         type: integer
+                      replicas:
+                        description: Replicas is the sum of sharding tables served
+                          on the current CN
+                        type: integer
                     type: object
                 type: object
               phase:

deploy/crds/core.matrixorigin.io_cnclaimsets.yaml

@@ -232,6 +232,10 @@ spec:
                               type: integer
                             pipelines:
                               type: integer
+                            replicas:
+                              description: Replicas is the sum of sharding tables
+                                served on the current CN
+                              type: integer
                           type: object
                       type: object
                     phase:

docs/reference/api-reference.md

@@ -1734,5 +1734,6 @@ _Appears in:_
 | --- | --- | --- | --- |
 | `connections` _integer_ |  |  |  |
 | `pipelines` _integer_ |  |  |  |
+| `replicas` _integer_ | Replicas is the sum of sharding tables served on the current CN |  |  |
 
 

go.mod

@@ -1,20 +1,20 @@
 module github.com/matrixorigin/matrixone-operator
 
-go 1.21.5
+go 1.23.0
 
-toolchain go1.22.0
+toolchain go1.23.1
 
 require (
 	github.com/blang/semver/v4 v4.0.0
 	github.com/cespare/xxhash v1.1.0
 	github.com/go-errors/errors v1.5.1
 	github.com/go-logr/logr v1.2.4
 	github.com/go-logr/zapr v1.2.4
-	github.com/go-sql-driver/mysql v1.7.1
+	github.com/go-sql-driver/mysql v1.8.1
 	github.com/golang/mock v1.6.0
 	github.com/google/go-cmp v0.6.0
 	github.com/matrixorigin/controller-runtime v0.0.0-20240909085031-5f706d779ec6
-	github.com/matrixorigin/matrixone v0.7.1-0.20240509144206-2eeef9246a17
+	github.com/matrixorigin/matrixone v0.7.1-0.20241008071244-fa8af5c4dad4
 	github.com/matrixorigin/matrixone-operator/api v0.0.0-20220926063007-e629f86256d2
 	github.com/minio/minio-go/v7 v7.0.63
 	github.com/onsi/ginkgo v1.16.5
@@ -39,6 +39,7 @@ require (
 )
 
 require (
+	filippo.io/edwards25519 v1.1.0 // indirect
 	github.com/DataDog/zstd v1.5.0 // indirect
 	github.com/FastFilter/xorfilter v0.1.4 // indirect
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
@@ -75,6 +76,7 @@ require (
 	github.com/axiomhq/hyperloglog v0.0.0-20230201085229-3ddf4bad03dc // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bits-and-blooms/bitset v1.8.0 // indirect
+	github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 // indirect
 	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cockroachdb/errors v1.9.1 // indirect
@@ -127,7 +129,7 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.17.4 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
@@ -136,6 +138,7 @@ require (
 	github.com/lni/vfs v0.2.1-0.20220616104132-8852fd867376 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/matrixorigin/monlp v0.0.0-20240825091235-be436dc30e78 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/miekg/dns v1.1.53 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
@@ -147,12 +150,13 @@ require (
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/mschoch/smat v0.2.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/ncruces/go-dns v1.2.5 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect
 	github.com/panjf2000/ants/v2 v2.7.4 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
-	github.com/pierrec/lz4/v4 v4.1.18 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/plar/go-adaptive-radix-tree v1.0.5 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -164,6 +168,8 @@ require (
 	github.com/rogpeppe/go-internal v1.10.0 // indirect
 	github.com/rs/xid v1.5.0 // indirect
 	github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
+	github.com/segmentio/asm v1.1.3 // indirect
+	github.com/segmentio/encoding v0.4.0 // indirect
 	github.com/shirou/gopsutil/v3 v3.23.12 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spf13/cobra v1.8.0 // indirect
@@ -190,24 +196,24 @@ require (
 	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/ratelimit v0.2.0 // indirect
-	golang.org/x/crypto v0.21.0 // indirect
-	golang.org/x/mod v0.16.0 // indirect
-	golang.org/x/net v0.22.0 // indirect
+	golang.org/x/crypto v0.23.0 // indirect
+	golang.org/x/mod v0.17.0 // indirect
+	golang.org/x/net v0.25.0 // indirect
 	golang.org/x/oauth2 v0.16.0 // indirect
-	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/sys v0.18.0 // indirect
-	golang.org/x/term v0.18.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
+	golang.org/x/term v0.20.0 // indirect
+	golang.org/x/text v0.17.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.14.0 // indirect
+	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
 	gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect
 	gonum.org/v1/gonum v0.14.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect
 	google.golang.org/grpc v1.62.1 // indirect
-	google.golang.org/protobuf v1.33.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
@@ -232,8 +238,8 @@ replace github.com/matrixorigin/matrixone-operator/api => ./api
 replace github.com/hashicorp/memberlist => github.com/matrixorigin/memberlist v0.5.1-0.20230322082342-95015c95ee76
 
 replace (
-	github.com/fagongzi/goetty/v2 v2.0.3-0.20230628075727-26c9a2fd5fb8 => github.com/matrixorigin/goetty/v2 v2.0.0-20231122095211-6a25dc9130ca
-	github.com/lni/dragonboat/v4 v4.0.0-20220815145555-6f622e8bcbef => github.com/matrixorigin/dragonboat/v4 v4.0.0-20240312080931-1b40809d7cea
+	github.com/fagongzi/goetty/v2 v2.0.3-0.20230628075727-26c9a2fd5fb8 => github.com/matrixorigin/goetty/v2 v2.0.0-20240611082008-a4de209fff3d
+	github.com/lni/dragonboat/v4 v4.0.0-20220815145555-6f622e8bcbef => github.com/matrixorigin/dragonboat/v4 v4.0.0-20240911092336-c85cbba24527
 	github.com/lni/goutils v1.3.1-0.20220604063047-388d67b4dbc4 => github.com/matrixorigin/goutils v1.3.1-0.20220604063047-388d67b4dbc4
 	github.com/lni/vfs v0.2.1-0.20220616104132-8852fd867376 => github.com/matrixorigin/vfs v0.2.1-0.20220616104132-8852fd867376
 )