fix: run health check and try multiple RPCs in fork tests #708
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| --- | ||
| "chainlink-deployments-framework": patch | ||
| --- | ||
|
|
||
| fix: run health check and try multiple RPCs in fork tests |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -273,6 +273,8 @@ type CTFAnvilChainProviderConfig struct { | |
| // gas limits, or other Anvil-specific options. | ||
| DockerCmdParamsOverrides []string | ||
|
|
||
| ForkURLs []string | ||
|
|
||
| // Optional: Port specifies the port for the Anvil container. If not provided, | ||
| // a free port will be automatically allocated. Use this when you need the Anvil | ||
| // instance to run on a specific port. | ||
|
|
@@ -538,6 +540,7 @@ func (p *CTFAnvilChainProvider) Cleanup(ctx context.Context) error { | |
| // Returns the external HTTP URL that can be used to connect to the Anvil node. | ||
| func (p *CTFAnvilChainProvider) startContainer(ctx context.Context, chainID string) (string, error) { | ||
| attempts := uint(10) | ||
| attempt := uint(0) | ||
|
|
||
| err := framework.DefaultNetwork(p.config.Once) | ||
| if err != nil { | ||
|
|
@@ -564,6 +567,11 @@ func (p *CTFAnvilChainProvider) startContainer(ctx context.Context, chainID stri | |
| portStr = strconv.Itoa(port) | ||
| } | ||
|
|
||
| if len(p.config.ForkURLs) > 0 { | ||
| url := p.config.ForkURLs[attempt%uint(len(p.config.ForkURLs))] | ||
| p.config.DockerCmdParamsOverrides = append(p.config.DockerCmdParamsOverrides, "--fork-url", url) | ||
|
Collaborator
There was a problem hiding this comment. so everytime this runs, we will append another --fork-url into
Contributor
Author
There was a problem hiding this comment. ooops; good catch. The earlier tests didn't have the health check, so this code path wasn't re-tested |
||
| } | ||
|
Comment on lines
+570
to
+573
There was a problem hiding this comment. The DockerCmdParamsOverrides slice is being mutated on every retry attempt, causing --fork-url parameters to accumulate. Each retry will append new fork-url arguments without clearing previous ones. Store the base overrides separately and create a fresh slice for each attempt, or clear the fork-url entries before appending. |
||
|
|
||
| // Create the input for the Anvil blockchain network | ||
| input := &blockchain.Input{ | ||
| Type: blockchain.TypeAnvil, | ||
|
|
@@ -609,6 +617,7 @@ func (p *CTFAnvilChainProvider) startContainer(ctx context.Context, chainID stri | |
| retry.Attempts(attempts), | ||
| retry.Delay(1*time.Second), | ||
| retry.DelayType(retry.FixedDelay), | ||
| retry.OnRetry(func(a uint, err error) { attempt = a + 1 }), | ||
| ) | ||
| if err != nil { | ||
| return "", fmt.Errorf("failed to start CTF Anvil container after %d attempts: %w", attempts, err) | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -42,14 +42,6 @@ func Test_AnvilConfig_Validate(t *testing.T) { | |
| }, | ||
| wantErr: true, | ||
| }, | ||
| } | ||
|
|
||
| for _, tt := range tests { | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||||
|---|---|---|---|---|---|---|---|---|
|
|
@@ -16,6 +16,7 @@ import ( | |||||||
| "time" | ||||||||
|
|
||||||||
| "github.com/ethereum/go-ethereum/common/hexutil" | ||||||||
| "github.com/ethereum/go-ethereum/ethclient" | ||||||||
| "github.com/go-resty/resty/v2" | ||||||||
| "github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain" | ||||||||
| "github.com/smartcontractkit/freeport" | ||||||||
|
|
@@ -214,11 +215,12 @@ func newAnvilChains( | |||||||
| "failed to decode network metadata for chain selector %d: %w", chainSelector, errMeta, | ||||||||
| ) | ||||||||
| } | ||||||||
| forkURLs, err := selectPublicRPC(ctx, lggr, &metadata, network.ChainSelector, network.RPCs) | ||||||||
| if err != nil { | ||||||||
| lggr.Infof("Excluding chain with ID %d from environment: %s", chainID, err.Error()) | ||||||||
| continue | ||||||||
| } | ||||||||
| if err = metadata.AnvilConfig.Validate(); err != nil { | ||||||||
| lggr.Infof("Excluding chain with ID %d from environment due to failed anvil config validation: %s", chainID, err.Error()) | ||||||||
| continue | ||||||||
| } | ||||||||
|
|
@@ -232,10 +234,10 @@ func newAnvilChains( | |||||||
|
|
||||||||
| var signerGenerator evmprov.SignerGenerator | ||||||||
| if kmsConfig.KeyID != "" { | ||||||||
| var terr error | ||||||||
| signerGenerator, terr = evmprov.TransactorFromKMS(kmsConfig.KeyID, kmsConfig.KeyRegion, "") | ||||||||
| if terr != nil { | ||||||||
| return nil, fmt.Errorf("failed to create transactor from KMS: %w", terr) | ||||||||
| } | ||||||||
| } else { | ||||||||
| signerGenerator = evmprov.TransactorFromRaw(onchainConfig.EVM.DeployerKey) | ||||||||
|
|
@@ -252,16 +254,14 @@ func newAnvilChains( | |||||||
| } | ||||||||
|
|
||||||||
| config := evmprov.CTFAnvilChainProviderConfig{ | ||||||||
| Once: &once, | ||||||||
| ConfirmFunctor: evmprov.ConfirmFuncGeth(3 * time.Minute), | ||||||||
| DockerCmdParamsOverrides: []string{"--auto-impersonate"}, | ||||||||
| Image: metadata.AnvilConfig.Image, | ||||||||
| ForkURLs: forkURLs, | ||||||||
| DeployerTransactorGen: signerGenerator, | ||||||||
| T: testing.TB(&testing.T{}), | ||||||||
| Port: "", // let the provider choose a free port; this ensures retries are handled properly | ||||||||
| } | ||||||||
|
|
||||||||
| if blockNumber, ok := blockNumbers[chainSelector]; ok { | ||||||||
|
|
@@ -304,26 +304,48 @@ func newAnvilChains( | |||||||
| } | ||||||||
|
|
||||||||
| func selectPublicRPC( | ||||||||
| ctx context.Context, lggr logger.Logger, metadata *cfgnet.EVMMetadata, chainSelector uint64, rpcs []cfgnet.RPC, | ||||||||
| ) ([]string, error) { | ||||||||
| if metadata.AnvilConfig.ArchiveHTTPURL != "" && isPublicRPC(metadata.AnvilConfig.ArchiveHTTPURL) { | ||||||||
| return []string{metadata.AnvilConfig.ArchiveHTTPURL}, nil | ||||||||
| } | ||||||||
|
|
||||||||
| urls := []string{} | ||||||||
| for _, rpc := range rpcs { | ||||||||
| if isPublicRPC(rpc.HTTPURL) { | ||||||||
| err := runHealthCheck(ctx, rpc.HTTPURL) | ||||||||
|
Collaborator
There was a problem hiding this comment. i wonder if it is worth parallelizing here instead of doing sequential health check?
Contributor
Author
There was a problem hiding this comment. the health checks I ran failed fast, so I didn't bother. We can reassess once we get a bit more real world usage. |
||||||||
| if err != nil { | ||||||||
| lggr.Infow("rpc failed health check", "url", rpc.HTTPURL, "chainSelector", chainSelector) | ||||||||
| } else { | ||||||||
| lggr.Infow("selected rpc for fork environment", "url", rpc.HTTPURL, "chainSelector", chainSelector) | ||||||||
| urls = append(urls, rpc.HTTPURL) | ||||||||
| } | ||||||||
| } | ||||||||
| } | ||||||||
|
|
||||||||
| if len(urls) == 0 { | ||||||||
| return []string{}, fmt.Errorf("no public RPCs found for chain %d", chainSelector) | ||||||||
| } | ||||||||
|
|
||||||||
| return urls, nil | ||||||||
| } | ||||||||
|
|
||||||||
| var privateRpcRegexp = regexp.MustCompile(`^https?://(rpcs\.cldev\.sh|gap\-.*\.(prod|stage)\.cldev\.sh|.*\.tail[a-z0-9]+\.ts\.net)(?::\d+)?/`) | ||||||||
|
|
||||||||
| func isPublicRPC(url string) bool { | ||||||||
| return !privateRpcRegexp.MatchString(url) | ||||||||
| } | ||||||||
|
|
||||||||
| func runHealthCheck(ctx context.Context, rpcURL string) error { | ||||||||
| client, err := ethclient.DialContext(ctx, rpcURL) | ||||||||
| if err != nil { | ||||||||
| return fmt.Errorf("failed to connect to rpc %v: %w", rpcURL, err) | ||||||||
| } | ||||||||
|
There was a problem hiding this comment. The ethclient connection is never closed, creating a resource leak. Add
Suggested change
|
||||||||
|
|
||||||||
| _, err = client.BlockNumber(ctx) | ||||||||
| if err != nil { | ||||||||
| return fmt.Errorf("failed to retrieve block number: %w", err) | ||||||||
| } | ||||||||
|
|
||||||||
| return nil | ||||||||
| } | ||||||||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||||
|---|---|---|---|---|---|---|---|---|
|
|
@@ -9,6 +9,7 @@ import ( | |||||||
| "time" | ||||||||
|
|
||||||||
| "github.com/go-resty/resty/v2" | ||||||||
| "github.com/jarcoal/httpmock" | ||||||||
| "github.com/stretchr/testify/assert" | ||||||||
| "github.com/stretchr/testify/require" | ||||||||
|
|
||||||||
|
|
@@ -174,14 +175,18 @@ func Test_isPublicRPC(t *testing.T) { | |||||||
|
|
||||||||
| func Test_selectPublicRPC(t *testing.T) { | ||||||||
| t.Parallel() | ||||||||
| httpmock.Activate(t) | ||||||||
|
There was a problem hiding this comment. The httpmock.Activate(t) call activates mocking globally for all tests, but lacks a corresponding Deactivate call. This will cause mock responses to leak into other parallel tests. Add
Suggested change
|
||||||||
|
|
||||||||
| lggr := logger.Test(t) | ||||||||
| nosetup := func(t *testing.T) { t.Helper() } | ||||||||
|
|
||||||||
| tests := []struct { | ||||||||
| name string | ||||||||
| metadata *cfgnet.EVMMetadata | ||||||||
| chainSelector uint64 | ||||||||
| rpcs []cfgnet.RPC | ||||||||
| setup func(t *testing.T) | ||||||||
| want []string | ||||||||
| wantErr string | ||||||||
| }{ | ||||||||
| { | ||||||||
|
|
@@ -192,42 +197,52 @@ func Test_selectPublicRPC(t *testing.T) { | |||||||
| rpcs: []cfgnet.RPC{ | ||||||||
| {HTTPURL: "http://other.url"}, | ||||||||
| }, | ||||||||
| setup: nosetup, | ||||||||
| want: []string{"http://metadata.url"}, | ||||||||
| }, | ||||||||
| { | ||||||||
| name: "success: selects only health public rpcs", | ||||||||
| metadata: &cfgnet.EVMMetadata{AnvilConfig: &cfgnet.AnvilConfig{ | ||||||||
| ArchiveHTTPURL: "http://gap-rpc.prod.cldev.sh/ethereum/sepolia", | ||||||||
| }}, | ||||||||
| rpcs: []cfgnet.RPC{ | ||||||||
| {HTTPURL: "http://rpcs.cldev.sh/ethereum/sepolia"}, | ||||||||
| {HTTPURL: "http://public.rpc1.url"}, | ||||||||
| {HTTPURL: "http://public.rpc2.url"}, | ||||||||
| {HTTPURL: "http://public.rpc3.url"}, | ||||||||
| }, | ||||||||
| setup: func(t *testing.T) { | ||||||||
| t.Helper() | ||||||||
| httpmock.RegisterResponder("POST", "http://public.rpc1.url", | ||||||||
| httpmock.NewStringResponder(200, `{"jsonrpc":"2.0","id":1,"result":"0x123"}`)) | ||||||||
| httpmock.RegisterResponder("POST", "http://public.rpc3.url", | ||||||||
| httpmock.NewStringResponder(200, `{"jsonrpc":"2.0","id":1,"result":"0x456"}`)) | ||||||||
| }, | ||||||||
| want: []string{"http://public.rpc1.url", "http://public.rpc3.url"}, | ||||||||
| }, | ||||||||
| { | ||||||||
| name: "failure: no public or healthy rpcs found", | ||||||||
| metadata: &cfgnet.EVMMetadata{AnvilConfig: &cfgnet.AnvilConfig{ | ||||||||
| ArchiveHTTPURL: "http://gap-rpc.prod.cldev.sh/ethereum/sepolia", | ||||||||
| }}, | ||||||||
| rpcs: []cfgnet.RPC{ | ||||||||
| {HTTPURL: "http://rpcs.cldev.sh/ethereum/sepolia"}, | ||||||||
| {HTTPURL: "http://unhealthy.rpc.url"}, | ||||||||
| }, | ||||||||
| setup: nosetup, | ||||||||
| wantErr: "no public RPCs found for chain 0", | ||||||||
| }, | ||||||||
| } | ||||||||
| for _, tt := range tests { | ||||||||
| t.Run(tt.name, func(t *testing.T) { | ||||||||
| t.Parallel() | ||||||||
|
|
||||||||
| tt.setup(t) | ||||||||
| urls, err := selectPublicRPC(t.Context(), lggr, tt.metadata, tt.chainSelector, tt.rpcs) | ||||||||
|
|
||||||||
| if tt.wantErr == "" { | ||||||||
| require.NoError(t, err) | ||||||||
| require.Equal(t, tt.want, urls) | ||||||||
| } else { | ||||||||
| require.ErrorContains(t, err, tt.wantErr) | ||||||||
| } | ||||||||
|
|
||||||||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
should we add some godoc for this?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
yeah, let me add it