Skip to content

Commit 91a63ec

Browse files
smirasmira
committed
fix: issue with etcd watch cancel
Fixes #174 The fix is to switch to proper gRPC client to embedded etcd. More information: cosi-project/state-etcd#156 Signed-off-by: Andrey Smirnov <andrey.smirnov@siderolabs.com>
1 parent 7f58ea4 commit 91a63ec

File tree

7 files changed

+34
-11
lines changed

7 files changed

+34
-11
lines changed

‎client/go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ replace (
1313
require (
1414
github.com/adrg/xdg v0.4.0
1515
github.com/blang/semver v3.5.1+incompatible
16-
github.com/cosi-project/runtime v0.4.1
16+
github.com/cosi-project/runtime v0.4.2
1717
github.com/fatih/color v1.16.0
1818
github.com/google/uuid v1.6.0
1919
github.com/gosuri/uiprogress v0.0.1

‎client/go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@ github.com/containerd/go-cni v1.1.9 h1:ORi7P1dYzCwVM6XPN4n3CbkuOx/NZ2DOqy+SHRdo9
2626
github.com/containerd/go-cni v1.1.9/go.mod h1:XYrZJ1d5W6E2VOvjffL3IZq0Dz6bsVlERHbekNK90PM=
2727
github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl31EQbXALQ=
2828
github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw=
29-
github.com/cosi-project/runtime v0.4.1 h1:9lJWw5cl3Lz1qP32bl2vxAsJs6LM8KdUGLCc9t/EGqw=
30-
github.com/cosi-project/runtime v0.4.1/go.mod h1:eXVAHf9QzzSVblLUtHHPFOZ7JBuz+GypHbao1vw+SdQ=
29+
github.com/cosi-project/runtime v0.4.2 h1:kJkhorzDWierDDbXn1BDHS6iQ7ai9AdvQOnK5uG/g8g=
30+
github.com/cosi-project/runtime v0.4.2/go.mod h1:eXVAHf9QzzSVblLUtHHPFOZ7JBuz+GypHbao1vw+SdQ=
3131
github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM=
3232
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
3333
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

‎go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ require (
2727
github.com/blang/semver/v4 v4.0.0
2828
github.com/cenkalti/backoff/v4 v4.3.0
2929
github.com/containers/image v3.0.2+incompatible
30-
github.com/cosi-project/runtime v0.4.1
31-
github.com/cosi-project/state-etcd v0.2.8
30+
github.com/cosi-project/runtime v0.4.2
31+
github.com/cosi-project/state-etcd v0.2.9
3232
github.com/crewjam/saml v0.4.14
3333
github.com/dustin/go-humanize v1.0.1
3434
github.com/emicklei/dot v1.6.1

‎go.sum

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -118,10 +118,10 @@ github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr
118118
github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec=
119119
github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
120120
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
121-
github.com/cosi-project/runtime v0.4.1 h1:9lJWw5cl3Lz1qP32bl2vxAsJs6LM8KdUGLCc9t/EGqw=
122-
github.com/cosi-project/runtime v0.4.1/go.mod h1:eXVAHf9QzzSVblLUtHHPFOZ7JBuz+GypHbao1vw+SdQ=
123-
github.com/cosi-project/state-etcd v0.2.8 h1:77e3VoA+hw/CSoJH4ibKWwrJwY5pSbIdOli7pbgYBNQ=
124-
github.com/cosi-project/state-etcd v0.2.8/go.mod h1:P/QdrlDu1JTptfteGgt6ClR9BHnbpt5EXxPFMvg+RuY=
121+
github.com/cosi-project/runtime v0.4.2 h1:kJkhorzDWierDDbXn1BDHS6iQ7ai9AdvQOnK5uG/g8g=
122+
github.com/cosi-project/runtime v0.4.2/go.mod h1:eXVAHf9QzzSVblLUtHHPFOZ7JBuz+GypHbao1vw+SdQ=
123+
github.com/cosi-project/state-etcd v0.2.9 h1:jPYLHvQEjAksqPxku7gcfX7o+a0cw289dAQfr58lWfE=
124+
github.com/cosi-project/state-etcd v0.2.9/go.mod h1:9PAv45buB5kWEGB+TuAk8FZxyRYxcB3198W/JndzcbA=
125125
github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM=
126126
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
127127
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=

‎internal/backend/runtime/omni/state_etcd.go

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"encoding/hex"
1212
"errors"
1313
"fmt"
14+
"net"
1415
"net/url"
1516
"os"
1617
"time"
@@ -25,10 +26,10 @@ import (
2526
"go.etcd.io/etcd/client/pkg/v3/transport"
2627
clientv3 "go.etcd.io/etcd/client/v3"
2728
"go.etcd.io/etcd/server/v3/embed"
28-
"go.etcd.io/etcd/server/v3/etcdserver/api/v3client"
2929
"go.uber.org/zap"
3030
"google.golang.org/grpc"
3131

32+
"github.com/siderolabs/gen/xslices"
3233
"github.com/siderolabs/omni/client/pkg/constants"
3334
"github.com/siderolabs/omni/internal/backend/logging"
3435
"github.com/siderolabs/omni/internal/backend/runtime/keyprovider"
@@ -204,7 +205,22 @@ func getEmbeddedEtcdClient(ctx context.Context, params *config.EtcdParams, logge
204205
return errors.New("etcd failed to start")
205206
}
206207

207-
cli := v3client.New(embeddedServer.Server)
208+
cli, err := clientv3.New(clientv3.Config{
209+
Endpoints: xslices.Map(embeddedServer.Clients, func(l net.Listener) string { return l.Addr().String() }),
210+
DialTimeout: 5 * time.Second,
211+
DialOptions: []grpc.DialOption{
212+
grpc.WithBlock(),
213+
grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(constants.GRPCMaxMessageSize)),
214+
grpc.WithSharedWriteBuffer(true),
215+
},
216+
Logger: logger.WithOptions(
217+
// never enable debug logs for etcd client, they are too chatty
218+
zap.IncreaseLevel(zap.InfoLevel),
219+
).With(logging.Component("etcd_client")),
220+
})
221+
if err != nil {
222+
return err
223+
}
208224

209225
closer := func() error {
210226
if err = cli.Close(); err != nil && !errors.Is(err, context.Canceled) {
@@ -263,6 +279,10 @@ func getExternalEtcdClient(ctx context.Context, params *config.EtcdParams, logge
263279
grpc.WithSharedWriteBuffer(true),
264280
},
265281
TLS: tlsConfig,
282+
Logger: logger.WithOptions(
283+
// never enable debug logs for etcd client, they are too chatty
284+
zap.IncreaseLevel(zap.InfoLevel),
285+
).With(logging.Component("etcd_client")),
266286
})
267287
if err != nil {
268288
return err

‎internal/backend/runtime/omni/state_etcd_election_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ func TestEtcdElections(t *testing.T) {
4646
require.NoError(t, omni.GetEmbeddedEtcdClient(ctx, &config.EtcdParams{
4747
Embedded: true,
4848
EmbeddedDBPath: t.TempDir(),
49+
Endpoints: []string{"http://localhost:0"},
4950
}, logger, func(ctx context.Context, client *clientv3.Client) error {
5051
started := make(chan int)
5152
closed := make(chan error)

‎internal/backend/runtime/omni/state_etcd_test.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,7 @@ func TestEtcdInitialization(t *testing.T) {
130130
EmbeddedDBPath: etcdDir,
131131
PrivateKeySource: step.args.privateKeySource,
132132
PublicKeyFiles: step.args.publicKeyFiles,
133+
Endpoints: []string{"http://localhost:0"},
133134
},
134135
},
135136
}, zaptest.NewLogger(t), func(context.Context, namespaced.StateBuilder) error {
@@ -196,6 +197,7 @@ func TestEncryptDecrypt(t *testing.T) {
196197
EmbeddedDBPath: etcdDir,
197198
PrivateKeySource: step.args.privateKeySource,
198199
PublicKeyFiles: step.args.publicKeyFiles,
200+
Endpoints: []string{"http://localhost:0"},
199201
},
200202
},
201203
}, zaptest.NewLogger(t),

0 commit comments

Comments
 (0)