Skip to content

Commit

Permalink
observability stack and CLI
Browse files Browse the repository at this point in the history
  • Loading branch information
skudasov committed Oct 17, 2024
1 parent c331f11 commit db87ee7
Show file tree
Hide file tree
Showing 19 changed files with 2,798 additions and 63 deletions.
38 changes: 19 additions & 19 deletions examples/exampleProduct/smoke.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@ docker_cmd_params = ['--block-time=1']
chain_id = '31337'

[[blockchain_a.out.nodes]]
ws_url = 'ws://127.0.0.1:32991/tcp'
http_url = 'http://127.0.0.1:32991/tcp'
docker_internal_ws_url = 'ws://anvil-14acf:8545'
docker_internal_http_url = 'http://anvil-14acf:8545'
ws_url = 'ws://127.0.0.1:32943/tcp'
http_url = 'http://127.0.0.1:32943/tcp'
docker_internal_ws_url = 'ws://anvil-8fdb4:8545'
docker_internal_http_url = 'http://anvil-8fdb4:8545'

[blockchain_b]
type = 'anvil'
Expand All @@ -23,13 +23,13 @@ docker_cmd_params = ['--block-time=1']
chain_id = '31338'

[[blockchain_b.out.nodes]]
ws_url = 'ws://127.0.0.1:32992/tcp'
http_url = 'http://127.0.0.1:32992/tcp'
docker_internal_ws_url = 'ws://anvil-0f35c:8546'
docker_internal_http_url = 'http://anvil-0f35c:8546'
ws_url = 'ws://127.0.0.1:32792/tcp'
http_url = 'http://127.0.0.1:32792/tcp'
docker_internal_ws_url = 'ws://anvil-f7b69:8546'
docker_internal_http_url = 'http://anvil-f7b69:8546'

[data_provider]
port = 8080
port = 9111

[data_provider.out]
data_provider_urls = ['http://localhost:8080/mock1', 'http://localhost:8080/mock2']
Expand All @@ -47,19 +47,19 @@ port = '5432'
image = 'public.ecr.aws/chainlink/chainlink'
tag = 'v2.17.0'
port = '6688'
test_config_overrides = "\n\t[[EVM]]\n\tChainID = '31337'\n\tMinIncomingConfirmations = 1\n\tMinContractPayment = '0.0001 link'\n\n\t\n\t[[EVM.Nodes]]\n\tName = 'node-9e105-0'\n\tWsUrl = 'ws://anvil-14acf:8545'\n\tHttpUrl = 'http://anvil-14acf:8545'\n\tSendOnly = false\n\tOrder = 100\n\t\n\t\n\t[[EVM]]\n\tChainID = '31338'\n\tMinIncomingConfirmations = 1\n\tMinContractPayment = '0.0001 link'\n\n\t\n\t[[EVM.Nodes]]\n\tName = 'node-cccf3-0'\n\tWsUrl = 'ws://anvil-0f35c:8546'\n\tHttpUrl = 'http://anvil-0f35c:8546'\n\tSendOnly = false\n\tOrder = 100\n\t\n\t"
test_config_overrides = "\n\t[[EVM]]\n\tChainID = '31337'\n\tMinIncomingConfirmations = 1\n\tMinContractPayment = '0.0001 link'\n\n\t\n\t[[EVM.Nodes]]\n\tName = 'default'\n\tWsUrl = 'ws://anvil-8fdb4:8545'\n\tHttpUrl = 'http://anvil-8fdb4:8545'\n\t\n\t"
user_config_overrides = "[Log]\nLevel = 'debug'\n"
test_secrets_overrides = ''
user_secrets_overrides = ''

[clnode_1.out]
[clnode_1.out.node]
url = '127.0.0.1:32994'
docker_internal_url = 'http://clnode-0060b:6688'
url = '127.0.0.1:32945'
docker_internal_url = 'http://clnode-29120:6688'

[clnode_1.out.postgresql]
url = 'postgresql://chainlink:thispasswordislongenough@127.0.0.1:32993/chainlink?sslmode=disable'
docker_internal_url = 'postgresql://chainlink:thispasswordislongenough@postgresql-ee7c0:5432/chainlink?sslmode=disable'
url = 'postgresql://chainlink:thispasswordislongenough@127.0.0.1:32944/chainlink?sslmode=disable'
docker_internal_url = 'postgresql://chainlink:thispasswordislongenough@postgresql-6b995:5432/chainlink?sslmode=disable'

[clnode_2]
data_provider_url = 'http://localhost:8080/mock1'
Expand All @@ -74,16 +74,16 @@ port = '5433'
image = 'public.ecr.aws/chainlink/chainlink'
tag = 'v2.17.0'
port = '6689'
test_config_overrides = "\n\t[[EVM]]\n\tChainID = '31338'\n\tMinIncomingConfirmations = 1\n\tMinContractPayment = '0.0001 link'\n\n\t\n\t[[EVM.Nodes]]\n\tName = 'node-cccf3-0'\n\tWsUrl = 'ws://anvil-0f35c:8546'\n\tHttpUrl = 'http://anvil-0f35c:8546'\n\tSendOnly = false\n\tOrder = 100\n\t\n\t"
test_config_overrides = "\n\t[[EVM]]\n\tChainID = '31337'\n\tMinIncomingConfirmations = 1\n\tMinContractPayment = '0.0001 link'\n\n\t\n\t[[EVM.Nodes]]\n\tName = 'default'\n\tWsUrl = 'ws://anvil-8fdb4:8545'\n\tHttpUrl = 'http://anvil-8fdb4:8545'\n\t\n\t"
user_config_overrides = "[Log]\nLevel = 'debug'\n"
test_secrets_overrides = ''
user_secrets_overrides = ''

[clnode_2.out]
[clnode_2.out.node]
url = '127.0.0.1:32996'
docker_internal_url = 'http://clnode-b2248:6689'
url = '127.0.0.1:32947'
docker_internal_url = 'http://clnode-40e9a:6689'

[clnode_2.out.postgresql]
url = 'postgresql://chainlink:thispasswordislongenough@127.0.0.1:32995/chainlink?sslmode=disable'
docker_internal_url = 'postgresql://chainlink:thispasswordislongenough@postgresql-b600d:5433/chainlink?sslmode=disable'
url = 'postgresql://chainlink:thispasswordislongenough@127.0.0.1:32946/chainlink?sslmode=disable'
docker_internal_url = 'postgresql://chainlink:thispasswordislongenough@postgresql-83825:5433/chainlink?sslmode=disable'
47 changes: 14 additions & 33 deletions examples/exampleProduct/smoke_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,17 @@ import (
"github.com/smartcontractkit/chainlink-testing-framework/framework"
"github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain"
"github.com/smartcontractkit/chainlink-testing-framework/framework/components/clnode"
"github.com/smartcontractkit/chainlink-testing-framework/framework/components/dp"
"github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake"
"github.com/stretchr/testify/require"
"testing"
)

type Config struct {
BlockchainA *blockchain.Input `toml:"blockchain_a" validate:"required"`
BlockchainB *blockchain.Input `toml:"blockchain_b" validate:"required"`
DataProvider *dp.Input `toml:"data_provider" validate:"required"`
CLNodeOne *clnode.Input `toml:"clnode_1" validate:"required"`
CLNodeTwo *clnode.Input `toml:"clnode_2" validate:"required"`
BlockchainA *blockchain.Input `toml:"blockchain_a" validate:"required"`
BlockchainB *blockchain.Input `toml:"blockchain_b" validate:"required"`
FakeDataProvider *fake.Input `toml:"data_provider" validate:"required"`
CLNodeOne *clnode.Input `toml:"clnode_1" validate:"required"`
CLNodeTwo *clnode.Input `toml:"clnode_2" validate:"required"`
}

func TestMultiNodeMultiNetwork(t *testing.T) {
Expand All @@ -25,50 +25,31 @@ func TestMultiNodeMultiNetwork(t *testing.T) {

bcNodes1, err := blockchain.NewBlockchainNetwork(in.BlockchainA)
require.NoError(t, err)
bcNodes2, err := blockchain.NewBlockchainNetwork(in.BlockchainB)
require.NoError(t, err)

dpout, err := dp.NewMockedDataProvider(in.DataProvider)
dpout, err := fake.NewMockedDataProvider(in.FakeDataProvider)
require.NoError(t, err)
in.CLNodeOne.DataProviderURL = dpout.Urls[0]
in.CLNodeTwo.DataProviderURL = dpout.Urls[0]

networkA, err := clnode.NewNetworkCfg(&clnode.NetworkConfig{
MinIncomingConfirmations: 1,
MinContractPayment: "0.0001 link",
EVMNodes: []*clnode.EVMNode{
{
SendOnly: false,
Order: 100,
},
},
}, bcNodes1)
require.NoError(t, err)
networkB, err := clnode.NewNetworkCfg(&clnode.NetworkConfig{
MinIncomingConfirmations: 1,
MinContractPayment: "0.0001 link",
EVMNodes: []*clnode.EVMNode{
{
SendOnly: false,
Order: 100,
},
},
}, bcNodes2)
net, err := clnode.NewNetworkCfgOneNetworkAllNodes(bcNodes1)
require.NoError(t, err)

in.CLNodeOne.Node.TestConfigOverrides = networkA + networkB
in.CLNodeTwo.Node.TestConfigOverrides = networkB
in.CLNodeOne.Node.TestConfigOverrides = net
in.CLNodeTwo.Node.TestConfigOverrides = net

_, err = clnode.NewNode(in.CLNodeOne)
require.NoError(t, err)

_, err = clnode.NewNode(in.CLNodeTwo)
require.NoError(t, err)

fmt.Printf("Node %d: http://%s\n", 1, in.CLNodeOne.Out.Node.Url)
fmt.Printf("Node %d: http://%s\n", 2, in.CLNodeTwo.Out.Node.Url)

t.Run("test feature A1", func(t *testing.T) {
client := resty.New()
_, err := client.R().
Get("http://localhost:8080/mock1")
Get("http://localhost:9111/mock1")
require.NoError(t, err)
})
t.Run("test feature A2", func(t *testing.T) {
Expand Down
101 changes: 101 additions & 0 deletions framework/cmd/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
package main

import (
"fmt"
"github.com/smartcontractkit/chainlink-testing-framework/framework"
"github.com/urfave/cli/v2"
"log"
"os"
"os/exec"
)

func main() {
app := &cli.App{
Name: "ctf",
Usage: "Manage Docker containers, networks, and TOML files for CTF framework",
Commands: []*cli.Command{
{
Name: "clean",
Usage: "Remove Docker containers and networks with 'framework=ctf' label",
Action: func(c *cli.Context) error {
// Execute the bash command
err := cleanDockerResources()
if err != nil {
return fmt.Errorf("failed to clean Docker resources: %w", err)
}
return nil
},
},
{
Name: "observability",
Usage: "Process a TOML file, remove fields with '.out' keys",
Subcommands: []*cli.Command{
{
Name: "up",
Usage: "",
UsageText: "",
Description: "",
Action: func(c *cli.Context) error { return observabilityUp() },
},
{
Name: "down",
Usage: "",
UsageText: "",
Description: "",
Action: func(c *cli.Context) error { return observabilityDown() },
},
},
Action: func(c *cli.Context) error {
_ = c.String("file")
// TODO: might be useful?
return nil
},
},
},
}

err := app.Run(os.Args)
if err != nil {
log.Fatal(err)
}
}

func cleanDockerResources() error {
// Bash command for removing Docker containers and networks with "framework=ctf" label
cmd := exec.Command("bash", "-c", `
docker ps -aq --filter "label=framework=ctf" | xargs -r docker rm -f && \
docker network ls --filter "label=framework=ctf" -q | xargs -r docker network rm
`)
framework.L.Info().Str("Cmd", cmd.String()).Msg("Running command")
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("error running clean command: %s", string(output))
}
return nil
}

func observabilityUp() error {
cmd := exec.Command("bash", "-c", fmt.Sprintf(`
cd %s && \
docker compose up
`, framework.ObservabilityPath))
framework.L.Info().Str("Cmd", cmd.String()).Msg("Running command")
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("error running clean command: %s", string(output))
}
return nil
}

func observabilityDown() error {
cmd := exec.Command("bash", "-c", fmt.Sprintf(`
cd %s && \
docker compose down -v
`, framework.ObservabilityPath))
framework.L.Info().Str("Cmd", cmd.String()).Msg("Running command")
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("error running clean command: %s", string(output))
}
return nil
}
12 changes: 8 additions & 4 deletions framework/components/clnode/clnode.go
Original file line number Diff line number Diff line change
Expand Up @@ -151,12 +151,16 @@ func newNode(in *Input, pgOut *postgres.Output) (*NodeOut, error) {
FileMode: 0644,
},
},
//HostConfigModifier: func(hc *container.HostConfig) {
// hc.NetworkMode = "host"
// hc.PortBindings = framework.MapTheSamePort(bindPort)
//},
WaitingFor: wait.ForLog("Listening and serving HTTP").WithStartupTimeout(2 * time.Minute),
}
// TODO: this is complex, though, desired by developers because of static addresses and fast login
// TODO: skipping for now
//if in.HostNetworkEnabled {
//req.HostConfigModifier = func(hc *container.HostConfig) {
// hc.NetworkMode = "host"
// hc.PortBindings = framework.MapTheSamePort(bindPort)
//}
//}
c, err := tc.GenericContainer(ctx, tc.GenericContainerRequest{
ContainerRequest: req,
Started: true,
Expand Down
22 changes: 22 additions & 0 deletions framework/components/clnode/connect_network.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,3 +56,25 @@ func NewNetworkCfg(in *NetworkConfig, out *blockchain.Output) (string, error) {
fmt.Println(resultCfg)
return resultCfg, nil
}

func NewNetworkCfgOneNetworkAllNodes(out *blockchain.Output) (string, error) {
resultCfg, err := framework.RenderTemplate(`
[[EVM]]
ChainID = '{{.ChainID}}'
MinIncomingConfirmations = 1
MinContractPayment = '0.0001 link'
{{range .Nodes}}
[[EVM.Nodes]]
Name = 'default'
WsUrl = '{{.DockerInternalWSUrl}}'
HttpUrl = '{{.DockerInternalHTTPUrl}}'
{{end}}
`, out)
if err != nil {
return "", err
}
fmt.Println("Configuring networks for CL node based on blockchain outputs:")
fmt.Println(resultCfg)
return resultCfg, nil
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package dp
package fake

import (
"fmt"
Expand All @@ -19,7 +19,7 @@ type Output struct {
Urls []string `toml:"data_provider_urls"`
}

func Mock(path string, response gin.H, statusCode int) error {
func Fake(path string, response gin.H, statusCode int) error {
if MockService == nil {
return fmt.Errorf("mock service is not initialized, please set up NewMockedDataProvider in your tests")
}
Expand Down
8 changes: 6 additions & 2 deletions framework/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,21 @@ module github.com/smartcontractkit/chainlink-testing-framework/framework
go 1.22

require (
github.com/BurntSushi/toml v1.4.0
github.com/aws/aws-sdk-go-v2/config v1.27.39
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.33.3
github.com/davecgh/go-spew v1.1.1
github.com/docker/docker v27.1.1+incompatible
github.com/docker/go-connections v0.5.0
github.com/gin-gonic/gin v1.10.0
github.com/go-playground/validator/v10 v10.22.1
github.com/google/uuid v1.6.0
github.com/pelletier/go-toml/v2 v2.2.3
github.com/pkg/errors v0.9.1
github.com/rs/zerolog v1.33.0
github.com/stretchr/testify v1.9.0
github.com/testcontainers/testcontainers-go v0.33.0
github.com/urfave/cli/v2 v2.27.5
)

require (
Expand Down Expand Up @@ -42,19 +45,18 @@ require (
github.com/containerd/log v0.1.0 // indirect
github.com/containerd/platforms v0.2.1 // indirect
github.com/cpuguy83/dockercfg v0.3.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
github.com/creack/pty v1.1.21 // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/gabriel-vasile/mimetype v1.4.6 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/gin-gonic/gin v1.10.0 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-resty/resty/v2 v2.15.3 // indirect
github.com/goccy/go-json v0.10.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/json-iterator/go v1.1.12 // indirect
Expand All @@ -79,13 +81,15 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shirou/gopsutil/v3 v3.23.12 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.12 // indirect
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
go.opentelemetry.io/otel v1.24.0 // indirect
Expand Down
11 changes: 11 additions & 0 deletions framework/observability/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
## Observability tools
We have some observability tools we use with our harness, you can use them by calling
```
ctf observability up
```
Change your `Loki` config in your `.envrc` you use to run tests
```
export LOKI_TENANT_ID=promtail
export LOKI_URL=http://host.docker.internal:3030/loki/api/v1/push
```
Then check [Loki](http://localhost:3000/explore?panes=%7B%220EE%22:%7B%22datasource%22:%22P8E80F9AEF21F6940%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bjob%3D%5C%22ctf%5C%22%7D%22,%22queryType%22:%22range%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22P8E80F9AEF21F6940%22%7D,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-5m%22,%22to%22:%22now%22%7D%7D%7D&schemaVersion=1&orgId=1) logs
Loading

0 comments on commit db87ee7

Please sign in to comment.