Data race in `TestWatchResourceGroup`

[rleungx]

Flaky Test

Which jobs are failing

==================
WARNING: DATA RACE
Write at 0x00c003c14ab8 by goroutine 18392:
  github.com/tikv/pd/pkg/mcs/resourcemanager/server.(*ResourceGroup).PatchSettings()
      /home/runner/work/pd/pd/pkg/mcs/resourcemanager/server/resource_group.go:124 +0x459
  github.com/tikv/pd/pkg/mcs/resourcemanager/server.(*Manager).ModifyResourceGroup()
      /home/runner/work/pd/pd/pkg/mcs/resourcemanager/server/manager.go:269 +0x21c
  github.com/tikv/pd/pkg/mcs/resourcemanager/server.(*Service).ModifyResourceGroup()
      /home/runner/work/pd/pd/pkg/mcs/resourcemanager/server/grpc_service.go:154 +0x1f7
  github.com/pingcap/kvproto/pkg/resource_manager._ResourceManager_ModifyResourceGroup_Handler.func1()
      /home/runner/go/pkg/mod/github.com/pingcap/kvproto@v0.0.0-20240716095229-5f7ffec83ea7/pkg/resource_manager/resource_manager.pb.go:2356 +0x82
  github.com/grpc-ecosystem/go-grpc-prometheus.init.(*ServerMetrics).UnaryServerInterceptor.func3()
      /home/runner/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:107 +0xbb
  go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.ChainUnaryServer.func6.1()
      /home/runner/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.4.0/chain.go:48 +0xd0
  go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func4()
      /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/api/v3rpc/interceptor.go:71 +0x2c1
  go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.ChainUnaryServer.func6.1()
      /home/runner/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.4.0/chain.go:48 +0xd0
  go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func3()
      /home/runner/go/pkg/mod/go.etcd.io/etcd/server/v3@v3.5.15/etcdserver/api/v3rpc/interceptor.go:78 +0xc8
  go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.ChainUnaryServer.func6()
      /home/runner/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.4.0/chain.go:53 +0x22c
  github.com/pingcap/kvproto/pkg/resource_manager._ResourceManager_ModifyResourceGroup_Handler()
      /home/runner/go/pkg/mod/github.com/pingcap/kvproto@v0.0.0-20240716095229-5f7ffec83ea7/pkg/resource_manager/resource_manager.pb.go:2358 +0x1c3
  google.golang.org/grpc.(*Server).processUnaryRPC()
      /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:1386 +0x19ac
  google.golang.org/grpc.(*Server).handleStream()
      /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:1797 +0x1a51
  google.golang.org/grpc.(*Server).serveStreams.func2.1()
      /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:1027 +0x15c

Previous read at 0x00c003c14ab8 by goroutine 14143:
  github.com/tikv/pd/pkg/mcs/resourcemanager/server.(*Manager).backgroundMetricsFlush()
      /home/runner/work/pd/pd/pkg/mcs/resourcemanager/server/manager.go:479 +0x2316
  github.com/tikv/pd/pkg/mcs/resourcemanager/server.(*Manager).Init.func4()
      /home/runner/work/pd/pd/pkg/mcs/resourcemanager/server/manager.go:182 +0x4f

Goroutine 18392 (running) created at:
  google.golang.org/grpc.(*Server).serveStreams.func2()
      /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:1038 +0x224
  google.golang.org/grpc/internal/transport.(*http2Server).operateHeaders()
      /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_server.go:619 +0x3a01
  google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams()
      /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_server.go:660 +0x285
  google.golang.org/grpc.(*Server).serveStreams()
      /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:1021 +0x6bb
  google.golang.org/grpc.(*Server).handleRawConn.func1()
      /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:957 +0x86

Goroutine 14143 (running) created at:
  github.com/tikv/pd/pkg/mcs/resourcemanager/server.(*Manager).Init()
Goroutine 18392 (running) created at:
  google.golang.org/grpc.(*Server).serveStreams.func2()
      /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:1038 +0x224
  google.golang.org/grpc/internal/transport.(*http2Server).operateHeaders()
      /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_server.go:619 +0x3a01
  google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams()
      /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/internal/transport/http2_server.go:660 +0x285
  google.golang.org/grpc.(*Server).serveStreams()
      /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:1021 +0x6bb
  google.golang.org/grpc.(*Server).handleRawConn.func1()
      /home/runner/go/pkg/mod/google.golang.org/grpc@v1.62.1/server.go:957 +0x86

Goroutine 14143 (running) created at:
  github.com/tikv/pd/pkg/mcs/resourcemanager/server.(*Manager).Init()
      /home/runner/work/pd/pd/pkg/mcs/resourcemanager/server/manager.go:182 +0xe8c
  github.com/tikv/pd/pkg/mcs/resourcemanager/server.(*Manager).Init-fm()
      <autogenerated>:1 +0x47
  github.com/tikv/pd/server.(*Server).campaignLeader()
      /home/runner/work/pd/pd/server/server.go:1758 +0x1d17
  github.com/tikv/pd/server.(*Server).leaderLoop()
      /home/runner/work/pd/pd/server/server.go:1685 +0x1f07
  github.com/tikv/pd/server.(*Server).startServerLoop.func1()
      /home/runner/work/pd/pd/server/server.go:660 +0x33
==================

CI link

https://github.com/tikv/pd/actions/runs/10733779546/job/29767729755?pr=8370

Reason for failure (if possible)

Anything else