diff --git a/.golangci.yml b/.golangci.yml index 7e5fdffb5..3fc4c3b0b 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -147,6 +147,7 @@ linters: - gochecknoglobals - gocognit - gocyclo + - godot - godox - goerr113 - golint diff --git a/cmd/ascii.go b/cmd/ascii.go new file mode 100644 index 000000000..66fa716e0 --- /dev/null +++ b/cmd/ascii.go @@ -0,0 +1,80 @@ +// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package cmd + +func peersASCII() []string { + return []string{ + " ____ ", + " | __ \\ ", + " | |__) |___ ___ _ __ ___ ", + " | ___// _ \\ / _ \\| '__|/ __| ", + " | | | __/| __/| | \\__ \\ ", + " |_| \\___| \\___||_| |___/ ", + } +} + +func beaconASCII() []string { + return []string{ + " ____ ", + " | _ \\ ", + " | |_) | ___ __ _ ___ ___ _ __ ", + " | _ < / _ \\ / _` | / __|/ _ \\ | '_ \\ ", + " | |_) || __/| (_| || (__| (_) || | | | ", + " |____/ \\___| \\__,_| \\___|\\___/ |_| |_| ", + } +} + +func validatorASCII() []string { + return []string{ + " __ __ _ _ _ _ ", + " \\ \\ / / | |(_, | | | | ", + " \\ \\ / /__ _ | | _ __| | __ _ | |_ ___ _ __ ", + " \\ \\/ // _` || || | / _` | / _` || __|/ _ \\ | '__| ", + " \\ /| (_| || || || (_| || (_| || |_| (_) || | ", + " \\/ \\__,_||_||_| \\__,_| \\__,_| \\__|\\___/ |_| ", + } +} + +func categoryDefaultASCII() []string { + return []string{ + " ", + " ", + " ", + " ", + " ", + " ", + } +} + +func scoreAASCII() []string { + return []string{ + " ", + " /\\ ", + " / \\ ", + " / /\\ \\ ", + " / ____ \\ ", + "/_/ \\_\\", + } +} + +func scoreBASCII() []string { + return []string{ + " ____ ", + "| _ \\ ", + "| |_) | ", + "| _ < ", + "| |_) | ", + "|____/ ", + } +} + +func scoreCASCII() []string { + return []string{ + " ____ ", + " / ____| ", + "| | ", + "| | ", + "| |____ ", + " \\_____| ", + } +} diff --git a/cmd/cmd.go b/cmd/cmd.go index 7b49d6c95..1c0069146 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -45,6 +45,11 @@ func New() *cobra.Command { ), newCombineCmd(newCombineFunc), newAlphaCmd( + newTestCmd( + newTestPeersCmd(runTestPeers), + newTestBeaconCmd(runTestBeacon), + newTestValidatorCmd(runTestValidator), + ), newAddValidatorsCmd(runAddValidatorsSolo), newViewClusterManifestCmd(runViewClusterManifest), ), diff --git a/cmd/duration.go b/cmd/duration.go new file mode 100644 index 000000000..8f713811d --- /dev/null +++ b/cmd/duration.go @@ -0,0 +1,67 @@ +// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package cmd + +import ( + "encoding/json" + "strconv" + "time" + + "github.com/obolnetwork/charon/app/errors" +) + +type Duration struct { + time.Duration +} + +func (d Duration) MarshalJSON() ([]byte, error) { + res, err := json.Marshal(d.String()) + if err != nil { + return nil, errors.Wrap(err, "marshal json duration") + } + + return res, nil +} + +func (d *Duration) UnmarshalJSON(b []byte) error { + var v any + err := json.Unmarshal(b, &v) + if err != nil { + return errors.Wrap(err, "unmarshal json duration") + } + switch value := v.(type) { + case float64: + d.Duration = time.Duration(value) + case string: + d.Duration, err = time.ParseDuration(value) + if err != nil { + return errors.Wrap(err, "parse string time to duration") + } + default: + return errors.New("invalid json duration") + } + + return nil +} + +func (d Duration) MarshalText() ([]byte, error) { + return []byte(d.String()), nil +} + +func (d *Duration) UnmarshalText(b []byte) error { + strTime := string(b) + intTime, err := strconv.ParseInt(strTime, 10, 64) + switch { + case err == nil: + d.Duration = time.Duration(intTime) + case errors.Is(err, strconv.ErrSyntax): + d.Duration, err = time.ParseDuration(strTime) + if err != nil { + return errors.Wrap(err, "parse string time to duration") + } + default: + return errors.Wrap(err, "invalid text duration") + } + + return nil +} diff --git a/cmd/duration_test.go b/cmd/duration_test.go new file mode 100644 index 000000000..213113172 --- /dev/null +++ b/cmd/duration_test.go @@ -0,0 +1,296 @@ +// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package cmd_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/obolnetwork/charon/cmd" +) + +func TestDurationMarshalJSON(t *testing.T) { + tests := []struct { + name string + in cmd.Duration + expected []byte + expectedErr string + }{ + { + name: "millisecond", + in: cmd.Duration{time.Millisecond}, + expected: []byte("\"1ms\""), + expectedErr: "", + }, + { + name: "day", + in: cmd.Duration{24 * time.Hour}, + expected: []byte("\"24h0m0s\""), + expectedErr: "", + }, + { + name: "1000 nanoseconds", + in: cmd.Duration{1000 * time.Nanosecond}, + expected: []byte("\"1µs\""), + expectedErr: "", + }, + { + name: "60 seconds", + in: cmd.Duration{60 * time.Second}, + expected: []byte("\"1m0s\""), + expectedErr: "", + }, + { + name: "empty", + in: cmd.Duration{}, + expected: []byte("\"0s\""), + expectedErr: "", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + res, err := test.in.MarshalJSON() + if test.expectedErr != "" { + require.ErrorContains(t, err, test.expectedErr) + return + } + require.NoError(t, err) + require.Equal(t, test.expected, res) + }) + } +} + +func TestDurationUnmarshalJSON(t *testing.T) { + tests := []struct { + name string + in []byte + expected cmd.Duration + expectedErr string + }{ + { + name: "millisecond", + in: []byte("\"1ms\""), + expected: cmd.Duration{time.Millisecond}, + expectedErr: "", + }, + { + name: "day", + in: []byte("\"24h0m0s\""), + expected: cmd.Duration{24 * time.Hour}, + expectedErr: "", + }, + { + name: "1000 nanoseconds", + in: []byte("\"1µs\""), + expected: cmd.Duration{1000 * time.Nanosecond}, + expectedErr: "", + }, + { + name: "60 seconds", + in: []byte("\"1m0s\""), + expected: cmd.Duration{60 * time.Second}, + expectedErr: "", + }, + { + name: "zero", + in: []byte("\"0s\""), + expected: cmd.Duration{}, + expectedErr: "", + }, + { + name: "millisecond number", + in: []byte("1000000"), + expected: cmd.Duration{time.Millisecond}, + expectedErr: "", + }, + { + name: "day number", + in: []byte("86400000000000"), + expected: cmd.Duration{24 * time.Hour}, + expectedErr: "", + }, + { + name: "1000 nanoseconds number", + in: []byte("1000"), + expected: cmd.Duration{1000 * time.Nanosecond}, + expectedErr: "", + }, + { + name: "60 seconds number", + in: []byte("60000000000"), + expected: cmd.Duration{60 * time.Second}, + expectedErr: "", + }, + { + name: "zero number", + in: []byte("0"), + expected: cmd.Duration{}, + expectedErr: "", + }, + { + name: "text string", + in: []byte("\"second\""), + expected: cmd.Duration{}, + expectedErr: "parse string time to duration", + }, + { + name: "invalid json", + in: []byte("second"), + expected: cmd.Duration{}, + expectedErr: "unmarshal json duration", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var res cmd.Duration + err := res.UnmarshalJSON(test.in) + if test.expectedErr != "" { + require.ErrorContains(t, err, test.expectedErr) + return + } + require.NoError(t, err) + require.Equal(t, test.expected, res) + }) + } +} + +func TestDurationMarshalText(t *testing.T) { + tests := []struct { + name string + in cmd.Duration + expected []byte + expectedErr string + }{ + { + name: "millisecond", + in: cmd.Duration{time.Millisecond}, + expected: []byte("1ms"), + expectedErr: "", + }, + { + name: "day", + in: cmd.Duration{24 * time.Hour}, + expected: []byte("24h0m0s"), + expectedErr: "", + }, + { + name: "1000 nanoseconds", + in: cmd.Duration{1000 * time.Nanosecond}, + expected: []byte("1µs"), + expectedErr: "", + }, + { + name: "60 seconds", + in: cmd.Duration{60 * time.Second}, + expected: []byte("1m0s"), + expectedErr: "", + }, + { + name: "empty", + in: cmd.Duration{}, + expected: []byte("0s"), + expectedErr: "", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + res, err := test.in.MarshalText() + if test.expectedErr != "" { + require.ErrorContains(t, err, test.expectedErr) + return + } + require.NoError(t, err) + require.Equal(t, test.expected, res) + }) + } +} + +func TestDurationUnmarshalText(t *testing.T) { + tests := []struct { + name string + in []byte + expected cmd.Duration + expectedErr string + }{ + { + name: "millisecond", + in: []byte("1ms"), + expected: cmd.Duration{time.Millisecond}, + expectedErr: "", + }, + { + name: "day", + in: []byte("24h0m0s"), + expected: cmd.Duration{24 * time.Hour}, + expectedErr: "", + }, + { + name: "1000 nanoseconds", + in: []byte("1µs"), + expected: cmd.Duration{1000 * time.Nanosecond}, + expectedErr: "", + }, + { + name: "60 seconds", + in: []byte("1m0s"), + expected: cmd.Duration{60 * time.Second}, + expectedErr: "", + }, + { + name: "zero", + in: []byte("0s"), + expected: cmd.Duration{}, + expectedErr: "", + }, + { + name: "millisecond number", + in: []byte("1000000"), + expected: cmd.Duration{time.Millisecond}, + expectedErr: "", + }, + { + name: "day number", + in: []byte("86400000000000"), + expected: cmd.Duration{24 * time.Hour}, + expectedErr: "", + }, + { + name: "1000 nanoseconds number", + in: []byte("1000"), + expected: cmd.Duration{1000 * time.Nanosecond}, + expectedErr: "", + }, + { + name: "60 seconds number", + in: []byte("60000000000"), + expected: cmd.Duration{60 * time.Second}, + expectedErr: "", + }, + { + name: "zero number", + in: []byte("0"), + expected: cmd.Duration{}, + expectedErr: "", + }, + { + name: "text string", + in: []byte("second"), + expected: cmd.Duration{}, + expectedErr: "parse string time to duration", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var res cmd.Duration + err := res.UnmarshalText(test.in) + if test.expectedErr != "" { + require.ErrorContains(t, err, test.expectedErr) + return + } + require.NoError(t, err) + require.Equal(t, test.expected, res) + }) + } +} diff --git a/cmd/test.go b/cmd/test.go new file mode 100644 index 000000000..c8bcfc416 --- /dev/null +++ b/cmd/test.go @@ -0,0 +1,233 @@ +// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package cmd + +import ( + "fmt" + "io" + "os" + "sort" + "strings" + "time" + + "github.com/pelletier/go-toml/v2" + "github.com/spf13/cobra" + + "github.com/obolnetwork/charon/app/errors" +) + +type testConfig struct { + OutputToml string + Quiet bool + TestCases []string + Timeout time.Duration +} + +func newTestCmd(cmds ...*cobra.Command) *cobra.Command { + root := &cobra.Command{ + Use: "test", + Short: "Test subcommands provide test suite to evaluate current cluster setup", + Long: `Test subcommands provide test suite to evaluate current cluster setup. Currently there is support for peer connection tests, beacon node and validator API.`, + } + + root.AddCommand(cmds...) + + return root +} + +func bindTestFlags(cmd *cobra.Command, config *testConfig) { + cmd.Flags().StringVar(&config.OutputToml, "output-toml", "", "File path to which output can be written in TOML format.") + cmd.Flags().StringSliceVar(&config.TestCases, "test-cases", nil, "List of comma separated names of tests to be exeucted.") + cmd.Flags().DurationVar(&config.Timeout, "timeout", 5*time.Minute, "Execution timeout for all tests.") + cmd.Flags().BoolVar(&config.Quiet, "quiet", false, "Do not print test results to stdout.") +} + +func mustOutputToFileOnQuiet(cmd *cobra.Command) error { + if cmd.Flag("quiet").Changed && !cmd.Flag("output-toml").Changed { + return errors.New("on --quiet, an --output-toml is required") + } + + return nil +} + +type testVerdict string + +const ( + // boolean tests + testVerdictOk testVerdict = "OK" + + // measurement tests + testVerdictGood testVerdict = "Good" + testVerdictAvg testVerdict = "Avg" + testVerdictBad testVerdict = "Bad" + + // failed tests + testVerdictFail testVerdict = "Fail" +) + +type categoryScore string + +const ( + categoryScoreA categoryScore = "A" + categoryScoreB categoryScore = "B" + categoryScoreC categoryScore = "C" +) + +type testResult struct { + Name string + Verdict testVerdict + Measurement string + Suggestion string + Error string +} + +type testCaseName struct { + name string + order uint +} + +type testCategoryResult struct { + CategoryName string + Targets map[string][]testResult + ExecutionTime Duration + Score categoryScore +} + +func appendScore(cat []string, score []string) []string { + var res []string + for i, l := range cat { + res = append(res, l+score[i]) + } + + return res +} + +func writeResultToFile(res testCategoryResult, path string) error { + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o444) + if err != nil { + return errors.Wrap(err, "create/open file") + } + defer f.Close() + err = toml.NewEncoder(f).Encode(res) + if err != nil { + return errors.Wrap(err, "encode testCategoryResult to TOML") + } + + return nil +} + +func writeResultToWriter(res testCategoryResult, w io.Writer) error { + var lines []string + + switch res.CategoryName { + case "peers": + lines = append(lines, peersASCII()...) + case "beacon": + lines = append(lines, beaconASCII()...) + case "validator": + lines = append(lines, validatorASCII()...) + default: + lines = append(lines, categoryDefaultASCII()...) + } + + switch res.Score { + case categoryScoreA: + lines = appendScore(lines, scoreAASCII()) + case categoryScoreB: + lines = appendScore(lines, scoreBASCII()) + case categoryScoreC: + lines = appendScore(lines, scoreCASCII()) + } + + lines = append(lines, "") + lines = append(lines, fmt.Sprintf("%-60s%s", "TEST NAME", "RESULT")) + suggestions := []string{} + for target, testResults := range res.Targets { + if target != "" && len(testResults) > 0 { + lines = append(lines, "") + lines = append(lines, target) + } + for _, singleTestRes := range testResults { + testOutput := "" + testOutput += fmt.Sprintf("%-60s", singleTestRes.Name) + if singleTestRes.Measurement != "" { + testOutput = strings.TrimSuffix(testOutput, strings.Repeat(" ", len(singleTestRes.Measurement)+1)) + testOutput = testOutput + singleTestRes.Measurement + " " + } + testOutput += string(singleTestRes.Verdict) + + if singleTestRes.Suggestion != "" { + suggestions = append(suggestions, singleTestRes.Suggestion) + } + + if singleTestRes.Error != "" { + testOutput += " - " + singleTestRes.Error + } + lines = append(lines, testOutput) + } + } + if len(suggestions) != 0 { + lines = append(lines, "") + lines = append(lines, "SUGGESTED IMPROVEMENTS") + lines = append(lines, suggestions...) + } + + lines = append(lines, "") + lines = append(lines, res.ExecutionTime.String()) + + lines = append(lines, "") + for _, l := range lines { + _, err := w.Write([]byte(l + "\n")) + if err != nil { + return err + } + } + + return nil +} + +func calculateScore(results []testResult) categoryScore { + // TODO(kalo): calculate score more elaborately (potentially use weights) + avg := 0 + for _, t := range results { + switch t.Verdict { + case testVerdictBad, testVerdictFail: + return categoryScoreC + case testVerdictGood: + avg++ + case testVerdictAvg: + avg-- + case testVerdictOk: + continue + } + } + + if avg < 0 { + return categoryScoreB + } + + return categoryScoreA +} + +func filterTests(supportedTestCases []testCaseName, cfg testConfig) []testCaseName { + if cfg.TestCases == nil { + return supportedTestCases + } + var filteredTests []testCaseName + for _, tc := range cfg.TestCases { + for _, stc := range supportedTestCases { + if stc.name == tc { + filteredTests = append(filteredTests, stc) + continue + } + } + } + + return filteredTests +} + +func sortTests(tests []testCaseName) { + sort.Slice(tests, func(i, j int) bool { + return tests[i].order < tests[j].order + }) +} diff --git a/cmd/testbeacon.go b/cmd/testbeacon.go new file mode 100644 index 000000000..cd36ab871 --- /dev/null +++ b/cmd/testbeacon.go @@ -0,0 +1,206 @@ +// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package cmd + +import ( + "context" + "io" + "sort" + "time" + + "github.com/spf13/cobra" + "golang.org/x/exp/maps" + + "github.com/obolnetwork/charon/app/errors" +) + +type testBeaconConfig struct { + testConfig + Endpoints []string +} + +func newTestBeaconCmd(runFunc func(context.Context, io.Writer, testBeaconConfig) error) *cobra.Command { + var config testBeaconConfig + + cmd := &cobra.Command{ + Use: "beacon", + Short: "Run multiple tests towards beacon nodes", + Long: `Run multiple tests towards beacon nodes. Verify that Charon can efficiently interact with Beacon Node(s).`, + Args: cobra.NoArgs, + PreRunE: func(cmd *cobra.Command, _ []string) error { + return mustOutputToFileOnQuiet(cmd) + }, + RunE: func(cmd *cobra.Command, _ []string) error { + return runFunc(cmd.Context(), cmd.OutOrStdout(), config) + }, + } + + bindTestFlags(cmd, &config.testConfig) + bindTestBeaconFlags(cmd, &config) + + return cmd +} + +func bindTestBeaconFlags(cmd *cobra.Command, config *testBeaconConfig) { + const endpoints = "endpoints" + cmd.Flags().StringSliceVar(&config.Endpoints, endpoints, nil, "[REQUIRED] Comma separated list of one or more beacon node endpoint URLs.") + mustMarkFlagRequired(cmd, endpoints) +} + +func supportedBeaconTestCases() map[testCaseName]func(context.Context, *testBeaconConfig, string) testResult { + return map[testCaseName]func(context.Context, *testBeaconConfig, string) testResult{ + {name: "ping", order: 1}: beaconPing, + } +} + +func runTestBeacon(ctx context.Context, w io.Writer, cfg testBeaconConfig) (err error) { + testCases := supportedBeaconTestCases() + queuedTests := filterTests(maps.Keys(testCases), cfg.testConfig) + if len(queuedTests) == 0 { + return errors.New("test case not supported") + } + sortTests(queuedTests) + sort.Slice(queuedTests, func(i, j int) bool { + return queuedTests[i].order < queuedTests[j].order + }) + + parentCtx := ctx + if parentCtx == nil { + parentCtx = context.Background() + } + timeoutCtx, cancel := context.WithTimeout(parentCtx, cfg.Timeout) + defer cancel() + + ch := make(chan map[string][]testResult) + testResults := make(map[string][]testResult) + startTime := time.Now() + finished := false + // run test suite for all beacon nodes + go testAllBeacons(timeoutCtx, queuedTests, testCases, cfg, ch) + + for !finished { + select { + case <-ctx.Done(): + finished = true + case result, ok := <-ch: + if !ok { + finished = true + } + maps.Copy(testResults, result) + } + } + execTime := Duration{time.Since(startTime)} + + // use highest score as score of all + var score categoryScore + for _, t := range testResults { + targetScore := calculateScore(t) + if score == "" || score > targetScore { + score = targetScore + } + } + + res := testCategoryResult{ + CategoryName: "beacon", + Targets: testResults, + ExecutionTime: execTime, + Score: score, + } + + if !cfg.Quiet { + err = writeResultToWriter(res, w) + if err != nil { + return err + } + } + + if cfg.OutputToml != "" { + err = writeResultToFile(res, cfg.OutputToml) + if err != nil { + return err + } + } + + return nil +} + +func testAllBeacons(ctx context.Context, queuedTestCases []testCaseName, allTestCases map[testCaseName]func(context.Context, *testBeaconConfig, string) testResult, cfg testBeaconConfig, resCh chan map[string][]testResult) { + defer close(resCh) + // run tests for all beacon nodes + res := make(map[string][]testResult) + chs := []chan map[string][]testResult{} + for _, enr := range cfg.Endpoints { + ch := make(chan map[string][]testResult) + chs = append(chs, ch) + go testSingleBeacon(ctx, queuedTestCases, allTestCases, cfg, enr, ch) + } + + for _, ch := range chs { + for { + // we are checking for context done (timeout) inside the go routine + result, ok := <-ch + if !ok { + break + } + maps.Copy(res, result) + } + } + + resCh <- res +} + +func testSingleBeacon(ctx context.Context, queuedTestCases []testCaseName, allTestCases map[testCaseName]func(context.Context, *testBeaconConfig, string) testResult, cfg testBeaconConfig, target string, resCh chan map[string][]testResult) { + defer close(resCh) + ch := make(chan testResult) + res := []testResult{} + // run all beacon tests for a beacon node, pushing each completed test to the channel until all are complete or timeout occurs + go runBeaconTest(ctx, queuedTestCases, allTestCases, cfg, target, ch) + + testCounter := 0 + finished := false + for !finished { + var name string + select { + case <-ctx.Done(): + name = queuedTestCases[testCounter].name + res = append(res, testResult{Name: name, Verdict: testVerdictFail, Error: "timeout"}) + finished = true + case result, ok := <-ch: + if !ok { + finished = true + break + } + name = queuedTestCases[testCounter].name + testCounter++ + result.Name = name + res = append(res, result) + } + } + + resCh <- map[string][]testResult{target: res} +} + +func runBeaconTest(ctx context.Context, queuedTestCases []testCaseName, allTestCases map[testCaseName]func(context.Context, *testBeaconConfig, string) testResult, cfg testBeaconConfig, target string, ch chan testResult) { + defer close(ch) + for _, t := range queuedTestCases { + select { + case <-ctx.Done(): + return + default: + ch <- allTestCases[t](ctx, &cfg, target) + } + } +} + +func beaconPing(ctx context.Context, _ *testBeaconConfig, _ string) testResult { + // TODO(kalo): implement real ping + select { + case <-ctx.Done(): + return testResult{Verdict: testVerdictFail} + default: + return testResult{ + Verdict: testVerdictFail, + Error: errors.New("ping not implemented").Error(), + } + } +} diff --git a/cmd/testbeacon_internal_test.go b/cmd/testbeacon_internal_test.go new file mode 100644 index 000000000..a6e7d0d22 --- /dev/null +++ b/cmd/testbeacon_internal_test.go @@ -0,0 +1,210 @@ +// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package cmd + +import ( + "bytes" + "context" + "io" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +//go:generate go test . -run=TestBeaconTest -update + +//nolint:dupl // code is marked as duplicate currently, as we are testing the same test skeleton, ignore for now +func TestBeaconTest(t *testing.T) { + tests := []struct { + name string + config testBeaconConfig + expected testCategoryResult + expectedErr string + cleanup func(*testing.T, string) + }{ + { + name: "default scenario", + config: testBeaconConfig{ + testConfig: testConfig{ + OutputToml: "", + Quiet: false, + TestCases: nil, + Timeout: time.Minute, + }, + Endpoints: []string{"beacon-endpoint-1", "beacon-endpoint-2"}, + }, + expected: testCategoryResult{ + Targets: map[string][]testResult{ + "beacon-endpoint-1": {{Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}}, + "beacon-endpoint-2": {{Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}}, + }, + }, + expectedErr: "", + }, + { + name: "timeout", + config: testBeaconConfig{ + testConfig: testConfig{ + OutputToml: "", + Quiet: false, + TestCases: nil, + Timeout: time.Nanosecond, + }, + Endpoints: []string{"beacon-endpoint-1", "beacon-endpoint-2"}, + }, + expected: testCategoryResult{ + Targets: map[string][]testResult{ + "beacon-endpoint-1": {{Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "timeout"}}, + "beacon-endpoint-2": {{Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "timeout"}}, + }, + }, + expectedErr: "", + }, + { + name: "quiet", + config: testBeaconConfig{ + testConfig: testConfig{ + OutputToml: "", + Quiet: true, + TestCases: nil, + Timeout: time.Minute, + }, + Endpoints: []string{"beacon-endpoint-1", "beacon-endpoint-2"}, + }, + expected: testCategoryResult{ + Targets: map[string][]testResult{ + "beacon-endpoint-1": {{Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}}, + "beacon-endpoint-2": {{Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}}, + }, + }, + expectedErr: "", + }, + { + name: "unsupported test", + config: testBeaconConfig{ + testConfig: testConfig{ + OutputToml: "", + Quiet: false, + TestCases: []string{"notSupportedTest"}, + Timeout: time.Minute, + }, + Endpoints: []string{"beacon-endpoint-1", "beacon-endpoint-2"}, + }, + expected: testCategoryResult{}, + expectedErr: "test case not supported", + }, + { + name: "custom test cases", + config: testBeaconConfig{ + testConfig: testConfig{ + OutputToml: "", + Quiet: false, + TestCases: []string{"ping"}, + Timeout: time.Minute, + }, + Endpoints: []string{"beacon-endpoint-1", "beacon-endpoint-2"}, + }, + expected: testCategoryResult{ + Targets: map[string][]testResult{ + "beacon-endpoint-1": {{Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}}, + "beacon-endpoint-2": {{Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}}, + }, + }, + expectedErr: "", + }, + + { + name: "write to file", + config: testBeaconConfig{ + testConfig: testConfig{ + OutputToml: "./write-to-file-test.toml.tmp", + Quiet: false, + TestCases: nil, + Timeout: time.Minute, + }, + Endpoints: []string{"beacon-endpoint-1", "beacon-endpoint-2"}, + }, + expected: testCategoryResult{ + CategoryName: "beacon", + Targets: map[string][]testResult{ + "beacon-endpoint-1": {{Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}}, + "beacon-endpoint-2": {{Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}}, + }, + Score: categoryScoreC, + }, + expectedErr: "", + cleanup: func(t *testing.T, p string) { + t.Helper() + err := os.Remove(p) + require.NoError(t, err) + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var buf bytes.Buffer + ctx := context.Background() + err := runTestBeacon(ctx, &buf, test.config) + if test.expectedErr != "" { + require.ErrorContains(t, err, test.expectedErr) + return + } else { + require.NoError(t, err) + } + defer func() { + if test.cleanup != nil { + test.cleanup(t, test.config.OutputToml) + } + }() + + if test.config.Quiet { + require.Empty(t, buf.String()) + } else { + testWriteOut(t, test.expected, buf) + } + + if test.config.OutputToml != "" { + testWriteFile(t, test.expected, test.config.OutputToml) + } + }) + } +} + +func TestBeaconTestFlags(t *testing.T) { + tests := []struct { + name string + args []string + expectedErr string + }{ + { + name: "default scenario", + args: []string{"beacon", "--endpoints=\"test.endpoint\""}, + expectedErr: "", + }, + { + name: "no endpoints flag", + args: []string{"beacon"}, + expectedErr: "required flag(s) \"endpoints\" not set", + }, + { + name: "no output toml on quiet", + args: []string{"beacon", "--endpoints=\"test.endpoint\"", "--quiet"}, + expectedErr: "on --quiet, an --output-toml is required", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cmd := newAlphaCmd(newTestBeaconCmd(func(context.Context, io.Writer, testBeaconConfig) error { return nil })) + cmd.SetArgs(test.args) + err := cmd.Execute() + if test.expectedErr != "" { + require.ErrorContains(t, err, test.expectedErr) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/cmd/testpeers.go b/cmd/testpeers.go new file mode 100644 index 000000000..1d3d7fd2d --- /dev/null +++ b/cmd/testpeers.go @@ -0,0 +1,314 @@ +// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package cmd + +import ( + "context" + "io" + "math/rand" + "time" + + "github.com/spf13/cobra" + "golang.org/x/exp/maps" + + "github.com/obolnetwork/charon/app/errors" +) + +type testPeersConfig struct { + testConfig + ENRs []string + P2PRelays []string +} + +func newTestPeersCmd(runFunc func(context.Context, io.Writer, testPeersConfig) error) *cobra.Command { + var config testPeersConfig + + cmd := &cobra.Command{ + Use: "peers", + Short: "Run multiple tests towards peer nodes", + Long: `Run multiple tests towards peer nodes. Verify that Charon can efficiently interact with Validator Client.`, + Args: cobra.NoArgs, + PreRunE: func(cmd *cobra.Command, _ []string) error { + return mustOutputToFileOnQuiet(cmd) + }, + RunE: func(cmd *cobra.Command, _ []string) error { + return runFunc(cmd.Context(), cmd.OutOrStdout(), config) + }, + } + + bindTestFlags(cmd, &config.testConfig) + bindTestPeersFlags(cmd, &config) + + return cmd +} + +func bindTestPeersFlags(cmd *cobra.Command, config *testPeersConfig) { + const enrs = "enrs" + cmd.Flags().StringSliceVar(&config.ENRs, "enrs", nil, "[REQUIRED] Comma-separated list of each peer ENR address.") + cmd.Flags().StringSliceVar(&config.P2PRelays, "p2p-relays", []string{"https://0.relay.obol.tech,https://2.relay.obol.tech"}, "Comma-separated list of each peer P2P relay address.") + mustMarkFlagRequired(cmd, enrs) +} + +func supportedPeerTestCases() map[testCaseName]func(context.Context, *testPeersConfig, string) testResult { + return map[testCaseName]func(context.Context, *testPeersConfig, string) testResult{ + {name: "ping", order: 1}: peerPingTest, + {name: "pingMeasure", order: 2}: peerPingMeasureTest, + {name: "pingLoad", order: 3}: peerPingLoadTest, + } +} + +func supportedSelfTestCases() map[testCaseName]func(context.Context, *testPeersConfig) testResult { + return map[testCaseName]func(context.Context, *testPeersConfig) testResult{ + {name: "natOpen", order: 1}: natOpenTest, + } +} + +func runTestPeers(ctx context.Context, w io.Writer, cfg testPeersConfig) (err error) { + peerTestCases := supportedPeerTestCases() + queuedTestsPeer := filterTests(maps.Keys(peerTestCases), cfg.testConfig) + sortTests(queuedTestsPeer) + + selfTestCases := supportedSelfTestCases() + queuedTestsSelf := filterTests(maps.Keys(selfTestCases), cfg.testConfig) + sortTests(queuedTestsSelf) + + if len(queuedTestsPeer) == 0 && len(queuedTestsSelf) == 0 { + return errors.New("test case not supported") + } + + parentCtx := ctx + if parentCtx == nil { + parentCtx = context.Background() + } + timeoutCtx, cancel := context.WithTimeout(parentCtx, cfg.Timeout) + defer cancel() + + selfCh := make(chan map[string][]testResult) + peersCh := make(chan map[string][]testResult) + testResults := make(map[string][]testResult) + var peersFinished, selfFinished bool + + startTime := time.Now() + // run test suite for all peers and separate test suite for testing self + go testAllPeers(timeoutCtx, queuedTestsPeer, peerTestCases, cfg, peersCh) + go testSelf(timeoutCtx, queuedTestsSelf, selfTestCases, cfg, selfCh) + + for !peersFinished || !selfFinished { + select { + case <-ctx.Done(): + peersFinished = true + selfFinished = true + case result, ok := <-selfCh: + if !ok { + selfFinished = true + break + } + maps.Copy(testResults, result) + case result, ok := <-peersCh: + if !ok { + peersFinished = true + break + } + maps.Copy(testResults, result) + } + } + execTime := Duration{time.Since(startTime)} + + // use lowest score as score of all + var score categoryScore + for _, t := range testResults { + targetScore := calculateScore(t) + if score == "" || score < targetScore { + score = targetScore + } + } + + res := testCategoryResult{ + CategoryName: "peers", + Targets: testResults, + ExecutionTime: execTime, + Score: score, + } + + if !cfg.Quiet { + err = writeResultToWriter(res, w) + if err != nil { + return err + } + } + + if cfg.OutputToml != "" { + err = writeResultToFile(res, cfg.OutputToml) + if err != nil { + return err + } + } + + return nil +} + +func testAllPeers(ctx context.Context, queuedTestCases []testCaseName, allTestCases map[testCaseName]func(context.Context, *testPeersConfig, string) testResult, cfg testPeersConfig, resCh chan map[string][]testResult) { + defer close(resCh) + // run tests for all peer nodes + res := make(map[string][]testResult) + chs := []chan map[string][]testResult{} + for _, enr := range cfg.ENRs { + ch := make(chan map[string][]testResult) + chs = append(chs, ch) + go testSinglePeer(ctx, queuedTestCases, allTestCases, cfg, enr, ch) + } + + for _, ch := range chs { + for { + // we are checking for context done (timeout) inside the go routine + result, ok := <-ch + if !ok { + break + } + maps.Copy(res, result) + } + } + + resCh <- res +} + +func testSinglePeer(ctx context.Context, queuedTestCases []testCaseName, allTestCases map[testCaseName]func(context.Context, *testPeersConfig, string) testResult, cfg testPeersConfig, target string, resCh chan map[string][]testResult) { + defer close(resCh) + ch := make(chan testResult) + res := []testResult{} + // run all peers tests for a peer, pushing each completed test to the channel until all are complete or timeout occurs + go runPeerTest(ctx, queuedTestCases, allTestCases, cfg, target, ch) + + testCounter := 0 + finished := false + for !finished { + var name string + select { + case <-ctx.Done(): + name = queuedTestCases[testCounter].name + res = append(res, testResult{Name: name, Verdict: testVerdictFail, Error: "timeout"}) + + finished = true + case result, ok := <-ch: + if !ok { + finished = true + continue + } + name = queuedTestCases[testCounter].name + testCounter++ + result.Name = name + res = append(res, result) + } + } + + resCh <- map[string][]testResult{target: res} +} + +func runPeerTest(ctx context.Context, queuedTestCases []testCaseName, allTestCases map[testCaseName]func(context.Context, *testPeersConfig, string) testResult, cfg testPeersConfig, target string, ch chan testResult) { + defer close(ch) + for _, t := range queuedTestCases { + select { + case <-ctx.Done(): + return + default: + ch <- allTestCases[t](ctx, &cfg, target) + } + } +} + +func testSelf(ctx context.Context, queuedTestCases []testCaseName, allTestCases map[testCaseName]func(context.Context, *testPeersConfig) testResult, cfg testPeersConfig, resCh chan map[string][]testResult) { + defer close(resCh) + ch := make(chan testResult) + res := []testResult{} + go runSelfTest(ctx, queuedTestCases, allTestCases, cfg, ch) + + testCounter := 0 + finished := false + for !finished { + var name string + select { + case <-ctx.Done(): + name = queuedTestCases[testCounter].name + res = append(res, testResult{Name: name, Verdict: testVerdictFail}) + finished = true + case result, ok := <-ch: + if !ok { + finished = true + continue + } + name = queuedTestCases[testCounter].name + testCounter++ + result.Name = name + res = append(res, result) + } + } + + resCh <- map[string][]testResult{"self": res} +} + +func runSelfTest(ctx context.Context, queuedTestCases []testCaseName, allTestCases map[testCaseName]func(context.Context, *testPeersConfig) testResult, cfg testPeersConfig, ch chan testResult) { + defer close(ch) + for _, t := range queuedTestCases { + select { + case <-ctx.Done(): + return + default: + ch <- allTestCases[t](ctx, &cfg) + } + } +} + +func peerPingTest(ctx context.Context, _ *testPeersConfig, _ string) testResult { + // TODO(kalo): implement real ping + select { + case <-ctx.Done(): + return testResult{Verdict: testVerdictFail} + default: + return testResult{ + Verdict: testVerdictFail, + Error: errors.New("ping not implemented").Error(), + } + } +} + +func peerPingMeasureTest(ctx context.Context, _ *testPeersConfig, _ string) testResult { + // TODO(kalo): implement real ping measure + s := rand.Int31n(300) + 100 //nolint: gosec // it's only temporary to showcase timeouts + time.Sleep(time.Duration(s) * time.Millisecond) + select { + case <-ctx.Done(): + return testResult{Verdict: testVerdictFail} + default: + return testResult{ + Verdict: testVerdictFail, + Measurement: "10ms", + Error: errors.New("pingMeasure not implemented").Error(), + } + } +} + +func peerPingLoadTest(ctx context.Context, _ *testPeersConfig, _ string) testResult { + // TODO(kalo): implement real ping load + select { + case <-ctx.Done(): + return testResult{Verdict: testVerdictFail} + default: + return testResult{ + Verdict: testVerdictFail, + Error: errors.New("pingLoad not implemented").Error(), + } + } +} + +func natOpenTest(ctx context.Context, _ *testPeersConfig) testResult { + // TODO(kalo): implement real port check + select { + case <-ctx.Done(): + return testResult{Verdict: testVerdictFail} + default: + return testResult{ + Verdict: testVerdictFail, + Error: errors.New("natOpen not implemented").Error(), + } + } +} diff --git a/cmd/testpeers_internal_test.go b/cmd/testpeers_internal_test.go new file mode 100644 index 000000000..a64405a19 --- /dev/null +++ b/cmd/testpeers_internal_test.go @@ -0,0 +1,340 @@ +// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package cmd + +import ( + "bytes" + "context" + "io" + "math/rand" + "os" + "slices" + "strings" + "testing" + "time" + + "github.com/pelletier/go-toml/v2" + "github.com/stretchr/testify/require" + "golang.org/x/exp/maps" +) + +//go:generate go test . -run=TestPeersTest -update + +//nolint:dupl // code is marked as duplicate currently, as we are testing the same test skeleton, ignore for now +func TestPeersTest(t *testing.T) { + tests := []struct { + name string + config testPeersConfig + expected testCategoryResult + expectedErr string + cleanup func(*testing.T, string) + }{ + { + name: "default scenario", + config: testPeersConfig{ + testConfig: testConfig{ + OutputToml: "", + Quiet: false, + TestCases: nil, + Timeout: time.Minute, + }, + ENRs: []string{"enr:-1", "enr:-2", "enr:-3"}, + }, + expected: testCategoryResult{ + CategoryName: "peers", + Targets: map[string][]testResult{ + "self": { + {Name: "natOpen", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "natOpen not implemented"}, + }, + "enr:-1": { + {Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}, + {Name: "pingMeasure", Verdict: testVerdictFail, Measurement: "10ms", Suggestion: "", Error: "pingMeasure not implemented"}, + {Name: "pingLoad", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "pingLoad not implemented"}, + }, + "enr:-2": { + {Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}, + {Name: "pingMeasure", Verdict: testVerdictFail, Measurement: "10ms", Suggestion: "", Error: "pingMeasure not implemented"}, + {Name: "pingLoad", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "pingLoad not implemented"}, + }, + "enr:-3": { + {Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}, + {Name: "pingMeasure", Verdict: testVerdictFail, Measurement: "10ms", Suggestion: "", Error: "pingMeasure not implemented"}, + {Name: "pingLoad", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "pingLoad not implemented"}, + }, + }, + Score: categoryScoreC, + }, + expectedErr: "", + }, + { + name: "timeout", + config: testPeersConfig{ + testConfig: testConfig{ + OutputToml: "", + Quiet: false, + TestCases: nil, + Timeout: 100 * time.Millisecond, + }, + ENRs: []string{"enr:-1", "enr:-2", "enr:-3"}, + }, + expected: testCategoryResult{ + CategoryName: "peers", + Targets: map[string][]testResult{ + "self": { + {Name: "natOpen", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "natOpen not implemented"}, + }, + "enr:-1": { + {Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}, + {Name: "pingMeasure", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "timeout"}, + }, + "enr:-2": { + {Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}, + {Name: "pingMeasure", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "timeout"}, + }, + "enr:-3": { + {Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}, + {Name: "pingMeasure", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "timeout"}, + }, + }, + Score: categoryScoreC, + }, + expectedErr: "", + }, + { + name: "quiet", + config: testPeersConfig{ + testConfig: testConfig{ + OutputToml: "", + Quiet: true, + TestCases: nil, + Timeout: 24 * time.Hour, + }, + ENRs: []string{"enr:-1", "enr:-2", "enr:-3"}, + }, + expected: testCategoryResult{ + CategoryName: "peers", + Targets: map[string][]testResult{ + "self": { + {Name: "natOpen", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "natOpen not implemented"}, + }, + "enr:-1": { + {Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}, + {Name: "pingMeasure", Verdict: testVerdictFail, Measurement: "10ms", Suggestion: "", Error: "pingMeasure not implemented"}, + {Name: "pingLoad", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "pingLoad not implemented"}, + }, + "enr:-2": { + {Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}, + {Name: "pingMeasure", Verdict: testVerdictFail, Measurement: "10ms", Suggestion: "", Error: "pingMeasure not implemented"}, + {Name: "pingLoad", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "pingLoad not implemented"}, + }, + "enr:-3": { + {Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}, + {Name: "pingMeasure", Verdict: testVerdictFail, Measurement: "10ms", Suggestion: "", Error: "pingMeasure not implemented"}, + {Name: "pingLoad", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "pingLoad not implemented"}, + }, + }, + Score: categoryScoreC, + }, + expectedErr: "", + }, + { + name: "unsupported test", + config: testPeersConfig{ + testConfig: testConfig{ + OutputToml: "", + Quiet: false, + TestCases: []string{"notSupportedTest"}, + Timeout: 24 * time.Hour, + }, + ENRs: []string{"enr:-1", "enr:-2", "enr:-3"}, + }, + expected: testCategoryResult{}, + expectedErr: "test case not supported", + }, + { + name: "custom test cases", + config: testPeersConfig{ + testConfig: testConfig{ + OutputToml: "", + Quiet: false, + TestCases: []string{"ping"}, + Timeout: 24 * time.Hour, + }, + ENRs: []string{"enr:-1", "enr:-2", "enr:-3"}, + }, + expected: testCategoryResult{ + CategoryName: "peers", + Targets: map[string][]testResult{ + "enr:-1": { + {Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}, + }, + "enr:-2": { + {Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}, + }, + "enr:-3": { + {Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}, + }, + }, + Score: categoryScoreC, + }, + expectedErr: "", + }, + { + name: "write to file", + config: testPeersConfig{ + testConfig: testConfig{ + OutputToml: "./write-to-file-test.toml.tmp", + Quiet: false, + TestCases: nil, + Timeout: time.Duration(rand.Int31n(222)) * time.Hour, + }, + ENRs: []string{"enr:-1", "enr:-2", "enr:-3"}, + }, + expected: testCategoryResult{ + CategoryName: "peers", + Targets: map[string][]testResult{ + "self": { + {Name: "natOpen", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "natOpen not implemented"}, + }, + "enr:-1": { + {Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}, + {Name: "pingMeasure", Verdict: testVerdictFail, Measurement: "10ms", Suggestion: "", Error: "pingMeasure not implemented"}, + {Name: "pingLoad", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "pingLoad not implemented"}, + }, + "enr:-2": { + {Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}, + {Name: "pingMeasure", Verdict: testVerdictFail, Measurement: "10ms", Suggestion: "", Error: "pingMeasure not implemented"}, + {Name: "pingLoad", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "pingLoad not implemented"}, + }, + "enr:-3": { + {Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}, + {Name: "pingMeasure", Verdict: testVerdictFail, Measurement: "10ms", Suggestion: "", Error: "pingMeasure not implemented"}, + {Name: "pingLoad", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "pingLoad not implemented"}, + }, + }, + Score: categoryScoreC, + }, + expectedErr: "", + cleanup: func(t *testing.T, p string) { + t.Helper() + err := os.Remove(p) + require.NoError(t, err) + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var buf bytes.Buffer + ctx := context.Background() + err := runTestPeers(ctx, &buf, test.config) + if test.expectedErr != "" { + require.ErrorContains(t, err, test.expectedErr) + return + } else { + require.NoError(t, err) + } + defer func() { + if test.cleanup != nil { + test.cleanup(t, test.config.OutputToml) + } + }() + + if test.config.Quiet { + require.Empty(t, buf.String()) + } else { + testWriteOut(t, test.expected, buf) + } + + if test.config.OutputToml != "" { + testWriteFile(t, test.expected, test.config.OutputToml) + } + }) + } +} + +func TestPeersTestFlags(t *testing.T) { + tests := []struct { + name string + args []string + expectedErr string + }{ + { + name: "default scenario", + args: []string{"peers", "--enrs=\"test.endpoint\""}, + expectedErr: "", + }, + { + name: "no enrs flag", + args: []string{"peers"}, + expectedErr: "required flag(s) \"enrs\" not set", + }, + { + name: "no output toml on quiet", + args: []string{"peers", "--enrs=\"test.endpoint\"", "--quiet"}, + expectedErr: "on --quiet, an --output-toml is required", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cmd := newAlphaCmd(newTestPeersCmd(func(context.Context, io.Writer, testPeersConfig) error { return nil })) + cmd.SetArgs(test.args) + err := cmd.Execute() + if test.expectedErr != "" { + require.ErrorContains(t, err, test.expectedErr) + } else { + require.NoError(t, err) + } + }) + } +} + +func testWriteOut(t *testing.T, expectedRes testCategoryResult, buf bytes.Buffer) { + t.Helper() + bufTests := strings.Split(buf.String(), "\n") + bufTests = slices.Delete(bufTests, 0, 8) + bufTests = slices.Delete(bufTests, len(bufTests)-4, len(bufTests)) + + nTargets := len(maps.Keys(expectedRes.Targets)) + require.Len(t, bufTests, len(slices.Concat(maps.Values(expectedRes.Targets)...))+nTargets*2) + + for i := 0; i < nTargets; i++ { + bufTests = bufTests[1:] + target := strings.Trim(bufTests[0], " ") + bufTests = bufTests[1:] + for _, test := range expectedRes.Targets[target] { + name, res, exist := strings.Cut(bufTests[0], " ") + require.True(t, exist) + require.Equal(t, name, test.Name) + require.Contains(t, res, test.Verdict) + require.Contains(t, res, test.Measurement) + require.Contains(t, res, test.Suggestion) + require.Contains(t, res, test.Error) + bufTests = bufTests[1:] + } + } + + require.Empty(t, bufTests) +} + +func testWriteFile(t *testing.T, expectedRes testCategoryResult, path string) { + t.Helper() + file, err := os.ReadFile(path) + require.NoError(t, err) + var res testCategoryResult + err = toml.Unmarshal(file, &res) + require.NoError(t, err) + + require.Equal(t, expectedRes.CategoryName, res.CategoryName) + require.Equal(t, expectedRes.Score, res.Score) + require.Equal(t, len(expectedRes.Targets), len(res.Targets)) + for targetName, testResults := range res.Targets { + for idx, testRes := range testResults { + require.Equal(t, expectedRes.Targets[targetName][idx].Verdict, testRes.Verdict) + require.Equal(t, expectedRes.Targets[targetName][idx].Verdict, testRes.Verdict) + require.Equal(t, expectedRes.Targets[targetName][idx].Measurement, testRes.Measurement) + require.Equal(t, expectedRes.Targets[targetName][idx].Suggestion, testRes.Suggestion) + require.Equal(t, expectedRes.Targets[targetName][idx].Error, testRes.Error) + } + } +} diff --git a/cmd/testvalidator.go b/cmd/testvalidator.go new file mode 100644 index 000000000..8ef63fff5 --- /dev/null +++ b/cmd/testvalidator.go @@ -0,0 +1,180 @@ +// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package cmd + +import ( + "context" + "io" + "sort" + "time" + + "github.com/spf13/cobra" + "golang.org/x/exp/maps" + + "github.com/obolnetwork/charon/app/errors" +) + +type testValidatorConfig struct { + testConfig + APIAddress string +} + +func newTestValidatorCmd(runFunc func(context.Context, io.Writer, testValidatorConfig) error) *cobra.Command { + var config testValidatorConfig + + cmd := &cobra.Command{ + Use: "validator", + Short: "Run multiple tests towards validator client", + Long: `Run multiple tests towards validator client. Verify that Charon can efficiently interact with other Charon peer nodes.`, + Args: cobra.NoArgs, + PreRunE: func(cmd *cobra.Command, _ []string) error { + return mustOutputToFileOnQuiet(cmd) + }, + RunE: func(cmd *cobra.Command, _ []string) error { + return runFunc(cmd.Context(), cmd.OutOrStdout(), config) + }, + } + + bindTestFlags(cmd, &config.testConfig) + bindTestValidatorFlags(cmd, &config) + + return cmd +} + +func bindTestValidatorFlags(cmd *cobra.Command, config *testValidatorConfig) { + cmd.Flags().StringVar(&config.APIAddress, "validator-api-address", "127.0.0.1:3600", "Listening address (ip and port) for validator-facing traffic proxying the beacon-node API.") +} + +func supportedValidatorTestCases() map[testCaseName]func(context.Context, *testValidatorConfig) testResult { + return map[testCaseName]func(context.Context, *testValidatorConfig) testResult{ + {name: "ping", order: 1}: validatorPing, + } +} + +func runTestValidator(ctx context.Context, w io.Writer, cfg testValidatorConfig) (err error) { + testCases := supportedValidatorTestCases() + queuedTests := filterTests(maps.Keys(testCases), cfg.testConfig) + if len(queuedTests) == 0 { + return errors.New("test case not supported") + } + sortTests(queuedTests) + sort.Slice(queuedTests, func(i, j int) bool { + return queuedTests[i].order < queuedTests[j].order + }) + + parentCtx := ctx + if parentCtx == nil { + parentCtx = context.Background() + } + timeoutCtx, cancel := context.WithTimeout(parentCtx, cfg.Timeout) + defer cancel() + + ch := make(chan map[string][]testResult) + testResults := make(map[string][]testResult) + startTime := time.Now() + finished := false + + // run test suite for a single validator client + go testSingleValidator(timeoutCtx, queuedTests, testCases, cfg, ch) + + for !finished { + select { + case <-ctx.Done(): + finished = true + case result, ok := <-ch: + if !ok { + finished = true + } + maps.Copy(testResults, result) + } + } + execTime := Duration{time.Since(startTime)} + + // use highest score as score of all + var score categoryScore + for _, t := range testResults { + targetScore := calculateScore(t) + if score == "" || score > targetScore { + score = targetScore + } + } + + res := testCategoryResult{ + CategoryName: "validator", + Targets: testResults, + ExecutionTime: execTime, + Score: score, + } + + if !cfg.Quiet { + err = writeResultToWriter(res, w) + if err != nil { + return err + } + } + + if cfg.OutputToml != "" { + err = writeResultToFile(res, cfg.OutputToml) + if err != nil { + return err + } + } + + return nil +} + +func testSingleValidator(ctx context.Context, queuedTestCases []testCaseName, allTestCases map[testCaseName]func(context.Context, *testValidatorConfig) testResult, cfg testValidatorConfig, resCh chan map[string][]testResult) { + defer close(resCh) + ch := make(chan testResult) + res := []testResult{} + // run all validator tests for a validator client, pushing each completed test to the channel until all are complete or timeout occurs + go testValidator(ctx, queuedTestCases, allTestCases, cfg, ch) + + testCounter := 0 + finished := false + for !finished { + var name string + select { + case <-ctx.Done(): + name = queuedTestCases[testCounter].name + res = append(res, testResult{Name: name, Verdict: testVerdictFail, Error: "timeout"}) + finished = true + case result, ok := <-ch: + if !ok { + finished = true + break + } + name = queuedTestCases[testCounter].name + testCounter++ + result.Name = name + res = append(res, result) + } + } + + resCh <- map[string][]testResult{cfg.APIAddress: res} +} + +func testValidator(ctx context.Context, queuedTests []testCaseName, allTests map[testCaseName]func(context.Context, *testValidatorConfig) testResult, cfg testValidatorConfig, ch chan testResult) { + defer close(ch) + for _, t := range queuedTests { + select { + case <-ctx.Done(): + return + default: + ch <- allTests[t](ctx, &cfg) + } + } +} + +func validatorPing(ctx context.Context, _ *testValidatorConfig) testResult { + // TODO(kalo): implement real ping + select { + case <-ctx.Done(): + return testResult{Verdict: testVerdictFail} + default: + return testResult{ + Verdict: testVerdictFail, + Error: errors.New("ping not implemented").Error(), + } + } +} diff --git a/cmd/testvalidator_internal_test.go b/cmd/testvalidator_internal_test.go new file mode 100644 index 000000000..fbe73980c --- /dev/null +++ b/cmd/testvalidator_internal_test.go @@ -0,0 +1,199 @@ +// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package cmd + +import ( + "bytes" + "context" + "io" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +//go:generate go test . -run=TestValidatorTest -update + +func TestValidatorTest(t *testing.T) { + tests := []struct { + name string + config testValidatorConfig + expected testCategoryResult + expectedErr string + cleanup func(*testing.T, string) + }{ + { + name: "default scenario", + config: testValidatorConfig{ + testConfig: testConfig{ + OutputToml: "", + Quiet: false, + TestCases: nil, + Timeout: time.Minute, + }, + APIAddress: "validator-api-address", + }, + expected: testCategoryResult{ + Targets: map[string][]testResult{ + "validator-api-address": {{Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}}, + }, + }, + expectedErr: "", + }, + { + name: "timeout", + config: testValidatorConfig{ + testConfig: testConfig{ + OutputToml: "", + Quiet: false, + TestCases: nil, + Timeout: time.Nanosecond, + }, + APIAddress: "validator-api-address", + }, + expected: testCategoryResult{ + Targets: map[string][]testResult{ + "validator-api-address": {{Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "timeout"}}, + }, + }, + expectedErr: "", + }, + { + name: "quiet", + config: testValidatorConfig{ + testConfig: testConfig{ + OutputToml: "", + Quiet: true, + TestCases: nil, + Timeout: time.Minute, + }, + APIAddress: "validator-api-address", + }, + expected: testCategoryResult{ + Targets: map[string][]testResult{ + "validator-api-address": {{Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}}, + }, + }, + expectedErr: "", + }, + { + name: "unsupported test", + config: testValidatorConfig{ + testConfig: testConfig{ + OutputToml: "", + Quiet: false, + TestCases: []string{"notSupportedTest"}, + Timeout: time.Minute, + }, + APIAddress: "validator-api-address", + }, + expected: testCategoryResult{}, + expectedErr: "test case not supported", + }, + { + name: "custom test cases", + config: testValidatorConfig{ + testConfig: testConfig{ + OutputToml: "", + Quiet: false, + TestCases: []string{"ping"}, + Timeout: time.Minute, + }, + APIAddress: "validator-api-address", + }, + expected: testCategoryResult{ + Targets: map[string][]testResult{ + "validator-api-address": {{Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}}, + }, + }, + expectedErr: "", + }, + + { + name: "write to file", + config: testValidatorConfig{ + testConfig: testConfig{ + OutputToml: "./write-to-file-test.toml.tmp", + Quiet: false, + TestCases: nil, + Timeout: time.Minute, + }, + APIAddress: "validator-api-address", + }, + expected: testCategoryResult{ + CategoryName: "validator", + Targets: map[string][]testResult{ + "validator-api-address": {{Name: "ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: "ping not implemented"}}, + }, + Score: categoryScoreC, + }, + expectedErr: "", + cleanup: func(t *testing.T, p string) { + t.Helper() + err := os.Remove(p) + require.NoError(t, err) + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var buf bytes.Buffer + ctx := context.Background() + err := runTestValidator(ctx, &buf, test.config) + if test.expectedErr != "" { + require.ErrorContains(t, err, test.expectedErr) + return + } else { + require.NoError(t, err) + } + defer func() { + if test.cleanup != nil { + test.cleanup(t, test.config.OutputToml) + } + }() + + if test.config.Quiet { + require.Empty(t, buf.String()) + } else { + testWriteOut(t, test.expected, buf) + } + + if test.config.OutputToml != "" { + testWriteFile(t, test.expected, test.config.OutputToml) + } + }) + } +} + +func TestValidatorTestFlags(t *testing.T) { + tests := []struct { + name string + args []string + expectedErr string + }{ + { + name: "default scenario", + args: []string{"validator", "--validator-api-address=\"test.endpoint\""}, + expectedErr: "", + }, + { + name: "no output toml on quiet", + args: []string{"validator", "--validator-api-address=\"test.endpoint\"", "--quiet"}, + expectedErr: "on --quiet, an --output-toml is required", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cmd := newAlphaCmd(newTestValidatorCmd(func(context.Context, io.Writer, testValidatorConfig) error { return nil })) + cmd.SetArgs(test.args) + err := cmd.Execute() + if test.expectedErr != "" { + require.ErrorContains(t, err, test.expectedErr) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/go.mod b/go.mod index 5e689d347..04088cbba 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/libp2p/go-libp2p v0.33.2 github.com/libp2p/go-msgio v0.3.0 github.com/multiformats/go-multiaddr v0.12.3 + github.com/pelletier/go-toml/v2 v2.1.0 github.com/prometheus/client_golang v1.19.0 github.com/prometheus/client_model v0.6.1 github.com/protolambda/eth2-shuffle v1.1.0 @@ -40,6 +41,7 @@ require ( go.uber.org/goleak v1.3.0 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.21.0 + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 golang.org/x/sync v0.7.0 golang.org/x/term v0.18.0 golang.org/x/time v0.5.0 @@ -148,7 +150,6 @@ require ( github.com/opencontainers/image-spec v1.1.0 // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect - github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pkg/profile v1.7.0 // indirect @@ -179,7 +180,6 @@ require ( go.uber.org/fx v1.20.1 // indirect go.uber.org/mock v0.4.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect golang.org/x/mod v0.16.0 // indirect golang.org/x/net v0.22.0 // indirect golang.org/x/sys v0.18.0 // indirect