libhive: client build args and alternative dockerfiles

clients/go-ethereum/Dockerfile

@@ -1,47 +1,7 @@
-## By default, the geth is pulled from Docker Hub.
+ARG baseimage=ethereum/client-go
+ARG tag=latest
 
-ARG branch=latest
-FROM ethereum/client-go:$branch as builder
-
-## ----
-## Uncomment the steps below (and comment out the steps above!) to build go-ethereum
-## from local sources in the ./go-ethereum directory.
-
-# FROM golang:1-alpine as builder
-# ARG branch=master
-# ADD go-ethereum /go-ethereum
-# WORKDIR /go-ethereum
-# RUN apk add --update bash curl jq git make
-# RUN make geth
-# RUN mv ./build/bin/geth /usr/local/bin/geth
-
-## ----
-
-## ----
-## Uncomment the steps below (and comment out the steps above!) to build
-## go-ethereum from a git branch.
-
-# FROM alpine:latest as builder
-# ARG user=ethereum
-# ARG repo=go-ethereum
-# ARG branch=master
-# RUN \
-#   apk add --update bash curl jq go git make gcc musl-dev         \
-#         ca-certificates linux-headers                         && \
-#   git clone --depth 1 --branch $branch                           \
-#       https://github.com/$user/$repo                          && \
-#   (cd go-ethereum && make geth)                               && \
-#   (cd go-ethereum                                             && \
-#   echo "{}"                                                      \
-#   | jq ".+ {\"repo\":\"$(git config --get remote.origin.url)\"}" \
-#   | jq ".+ {\"branch\":\"$(git rev-parse --abbrev-ref HEAD)\"}"  \
-#   | jq ".+ {\"commit\":\"$(git rev-parse HEAD)\"}"               \
-#   > /version.json)                                            && \
-#   cp go-ethereum/build/bin/geth /usr/local/bin/geth           && \
-#   apk del go git make gcc musl-dev linux-headers              && \
-#   rm -rf /go-ethereum && rm -rf /var/cache/apk/*
-
-## ----
+FROM $baseimage:$tag as builder
 
 FROM alpine:latest
 RUN apk add --update bash curl jq

clients/go-ethereum/Dockerfile.git

@@ -0,0 +1,40 @@
+## Pulls geth from a git repository and builds it from source.
+
+FROM alpine:latest as builder
+ARG github=ethereum/go-ethereum
+ARG tag=master
+
+RUN \
+  apk add --update bash curl jq go git make gcc musl-dev              \
+        ca-certificates linux-headers                              && \
+  git clone --depth 1 --branch $tag https://github.com/$github     && \
+  cd go-ethereum                                                   && \
+  make geth                                                        && \
+  cp go-ethereum/build/bin/geth /usr/local/bin/geth                && \
+  apk del go git make gcc musl-dev linux-headers                   && \
+  rm -rf /go-ethereum && rm -rf /var/cache/apk/*
+
+FROM alpine:latest
+RUN apk add --update bash curl jq
+COPY --from=builder /usr/local/bin/geth /usr/local/bin/geth
+
+# Generate the version.txt file.
+RUN /usr/local/bin/geth console --exec 'console.log(admin.nodeInfo.name)' --maxpeers=0 --nodiscover --dev 2>/dev/null | head -1 > /version.txt
+
+# Inject the startup script.
+ADD geth.sh /geth.sh
+ADD mapper.jq /mapper.jq
+RUN chmod +x /geth.sh
+
+# Inject the enode id retriever script.
+RUN mkdir /hive-bin
+ADD enode.sh /hive-bin/enode.sh
+RUN chmod +x /hive-bin/enode.sh
+
+# Add a default genesis file.
+ADD genesis.json /genesis.json
+
+# Export the usual networking ports to allow outside access to the node
+EXPOSE 8545 8546 8547 8551 30303 30303/udp
+
+ENTRYPOINT ["/geth.sh"]

clients/go-ethereum/Dockerfile.local

@@ -0,0 +1,33 @@
+## Build geth from source from a local directory called go-ethereum.
+
+FROM golang:1-alpine as builder
+ADD go-ethereum /go-ethereum
+WORKDIR /go-ethereum
+RUN apk add --update bash curl jq git make
+RUN make geth
+RUN mv ./build/bin/geth /usr/local/bin/geth
+
+FROM alpine:latest
+RUN apk add --update bash curl jq
+COPY --from=builder /usr/local/bin/geth /usr/local/bin/geth
+
+# Generate the version.txt file.
+RUN /usr/local/bin/geth console --exec 'console.log(admin.nodeInfo.name)' --maxpeers=0 --nodiscover --dev 2>/dev/null | head -1 > /version.txt
+
+# Inject the startup script.
+ADD geth.sh /geth.sh
+ADD mapper.jq /mapper.jq
+RUN chmod +x /geth.sh
+
+# Inject the enode id retriever script.
+RUN mkdir /hive-bin
+ADD enode.sh /hive-bin/enode.sh
+RUN chmod +x /hive-bin/enode.sh
+
+# Add a default genesis file.
+ADD genesis.json /genesis.json
+
+# Export the usual networking ports to allow outside access to the node
+EXPOSE 8545 8546 8547 8551 30303 30303/udp
+
+ENTRYPOINT ["/geth.sh"]

docs/clients.md

@@ -8,20 +8,29 @@ Clients are docker images which can be instantiated by a simulation. A client de
 consists of a Dockerfile and associated resources. Client definitions live in
 subdirectories of `clients/` in the hive repository.
 
+See the [go-ethereum client definition][geth-docker] for an example of a client
+Dockerfile.
+
 When hive runs a simulation, it first builds all client docker images using their
 Dockerfile, i.e. it basically runs `docker build .` in the client directory. Since most
 client definitions wrap an existing Ethereum client, and building the client from source
 may take a long time, it is usually best to base the hive client wrapper on a pre-built
 docker image from Docker Hub.
 
-Client Dockerfiles should support an optional argument named `branch`, which specifies the
-requested client version. This argument can be set by users by appending it to the client
-name like:
+The client Dockerfile should support an optional argument named `branch`, which specifies
+the requested client version. This argument can be set by users by appending it to the
+client name like:
 
     ./hive --sim my-simulation --client go-ethereum_v1.9.23,go_ethereum_v1.9.22
 
-See the [go-ethereum client definition][geth-docker] for an example of a client
-Dockerfile.
+Other build arguments can also be set using a YAML file, see the [hive command
+documentation][hive-client-yaml] for more information.
+
+### Alternative Dockerfiles
+
+There can be other Dockerfiles besides the main one. Typically, a client should also
+provide a `Dockerfile.git` that builds the client from source code. Alternative
+Dockerfiles can be selected through hive's `-client-file` YAML configuration.
 
 ### hive.yaml
 
@@ -35,7 +44,7 @@ list:
 
 The role list is available to simulators and can be used to differentiate between clients
 based on features. Declaring a client role also signals that the client supports certain
-role-specific environment variables and files. If `hive.yml` is missing or doesn't declare
+role-specific environment variables and files. If `hive.yaml` is missing or doesn't declare
 roles, the `eth1` role is assumed.
 
 ### /version.txt
@@ -155,6 +164,7 @@ For the server role, the following additional variables should be supported:
 
 [LES]: https://github.com/ethereum/devp2p/blob/master/caps/les.md
 [geth-docker]: ../clients/go-ethereum/Dockerfile
+[hive-client-yaml]: ./commandline.md#client-build-parameters
 [oe-genesis-jq]: ../clients/openethereum/mapper.jq
 [EIP-155]: https://eips.ethereum.org/EIPS/eip-155
 [EIP-606]: https://eips.ethereum.org/EIPS/eip-606

docs/commandline.md

@@ -39,13 +39,42 @@ version by appending it to the client name with `_`, for example:
 
     ./hive --sim devp2p --client go-ethereum_v1.9.22,go-ethereum_v1.9.23
 
-Simulation runs can be customized in many ways. Here's an overview of the available
-command-line options.
+### Client Build Parameters
 
-`--client.checktimelimit <timeout>`: The timeout of waiting for clients to open up TCP
-port 8545. If a very long chain is imported, this timeout may need to be quite long. A
-lower value means that hive won't wait as long in case the node crashes and never opens
-the RPC port. Defaults to 3 minutes.
+The client list for a run can also be given in a YAML file. This also allows further
+customization of the build arguments of the client. Specify the `--client-file` option to
+use a client list file.
+
+    ./hive --sim my-simulation --client-file clients.yaml
+
+Here is an example clients.yaml file:
+
+    - client: go-ethereum
+      dockerfile: git
+    - client: nethermind
+      build_args:
+        baseimage: nethermindeth/hive
+        tag: latest
+
+For each client in the list, the following options can be given:
+
+ - `client`: Name of the client. This must refer to a known client in the clients/ directory.
+ - `dockerfile`: The Dockerfile extension to use. For example, specifying `git` here will
+   build the client using `Dockerfile.git` instead of the default `Dockerfile`.
+ - `nametag`: this can be used to assign a more descriptive name to the client. If unset,
+   a unique nametag will be chosen based on the version tag and/or build arguments.
+ - `build_args`: Build arguments passed to the Dockerfile, see below.
+
+Supported build arguments depend on the client and the docker image being used. Common build
+arguments are:
+
+ - `tag`: The git commit/tag/branch or docker tag name to use.
+ - `baseimage`: For clients pulled from DockerHub, this can be used to override the organization
+   and image name. Example `ethereum/client-go`.
+ - `github`: For client Dockerfiles building from git, this setting can be used to change
+   the source code repository (fork) on GitHub. Example: `ethereum/go-ethereum`.
+
+### Docker Options
 
 `--docker.pull`: Setting this option makes hive re-pull the base images of all built
 docker containers.
@@ -56,16 +85,7 @@ docker containers.
 rebuild. You can use this option during simulator development to ensure a new image is
 built even when there are no changes to the simulator code.
 
-`--sim.timelimit <timeout>`: Simulation timeout. Hive aborts the simulator if it exceeds
-this time. There is no default timeout.
-
-`--sim.loglevel <level>`: Selects log level of client instances. Supports values 0-5,
-defaults to 3. Note that this value may be overridden by simulators for specific clients.
-This sets the default value of `HIVE_LOGLEVEL` in client containers.
-
-`--sim.parallelism <number>`: Sets max number of parallel clients/containers. This is
-interpreted by simulators. It sets the `HIVE_PARALLELISM` environment variable. Defaults
-to 1.
+### Simulation Options
 
 `--sim.limit <pattern>`: Specifies a regular expression to selectively enable suites and
 test cases. This is interpreted by simulators. It sets the `HIVE_TEST_PATTERN` environment
@@ -86,6 +106,22 @@ directory (note the first `/`, matching any suite name):
 
     ./hive --sim ethereum/consensus --sim.limit /stBugs/
 
+`--sim.timelimit <timeout>`: Simulation timeout. Hive aborts the simulator if it exceeds
+this time. There is no default timeout.
+
+`--client.checktimelimit <timeout>`: The timeout of waiting for clients to open up TCP
+port 8545. If a very long chain is imported, this timeout may need to be quite long. A
+lower value means that hive won't wait as long in case the node crashes and never opens
+the RPC port. Defaults to 3 minutes.
+
+`--sim.loglevel <level>`: Selects log level of client instances. Supports values 0-5,
+defaults to 3. Note that this value may be overridden by simulators for specific clients.
+This sets the default value of `HIVE_LOGLEVEL` in client containers.
+
+`--sim.parallelism <number>`: Sets max number of parallel clients/containers. This is
+interpreted by simulators. It sets the `HIVE_PARALLELISM` environment variable. Defaults
+to 1.
+
 ## Viewing simulation results (hiveview)
 
 The results of hive simulation runs are stored in JSON files containing test results, and

go.mod

@@ -78,7 +78,7 @@ require (
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	golang.org/x/crypto v0.4.0 // indirect
-	golang.org/x/exp v0.0.0-20230206171751-46f607a40771 // indirect
+	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect
 	golang.org/x/sys v0.5.0 // indirect
 	golang.org/x/text v0.7.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect

go.sum

@@ -910,6 +910,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
 golang.org/x/exp v0.0.0-20230206171751-46f607a40771 h1:xP7rWLUr1e1n2xkK5YB4LI0hPEy3LJC6Wk+D4pGlOJg=
 golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
+golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
+golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

hive.go

@@ -34,11 +34,14 @@ func main() {
 		simDevModeAPIEndpoint = flag.String("dev.addr", "127.0.0.1:3000", "Endpoint that the simulator API listens on")
 		useCredHelper         = flag.Bool("docker.cred-helper", false, "configure docker authentication using locally-configured credential helper")
 
+		clientsFile = flag.String("client-file", "", `YAML `+"`file`"+` containing client configurations.`)
+
 		clients = flag.String("client", "go-ethereum", "Comma separated `list` of clients to use. Client names in the list may be given as\n"+
 			"just the client name, or a client_branch specifier. If a branch name is supplied,\n"+
 			"the client image will use the given git branch or docker tag. Multiple instances of\n"+
 			"a single client type may be requested with different branches.\n"+
-			"Example: \"besu_latest,besu_20.10.2\"")
+			"Example: \"besu_latest,besu_20.10.2\"\n")
+
 		clientTimeout = flag.Duration("client.checktimelimit", 3*time.Minute, "The `timeout` of waiting for clients to open up the RPC port.\n"+
 			"If a very long chain is imported, this timeout may need to be quite large.\n"+
 			"A lower value means that hive won't wait as long in case the node crashes and\n"+
@@ -111,8 +114,24 @@ func main() {
 		ClientStartTimeout: *clientTimeout,
 	}
 	runner := libhive.NewRunner(inv, builder, cb)
-	clientList := splitAndTrim(*clients, ",")
 
+	// Parse the client list.
+	// It can be supplied as a comma-separated list, or as a YAML file.
+	checkFlagsExclusive("client", "client-file")
+	var clientList []libhive.ClientDesignator
+	if *clientsFile != "" {
+		clientList, err = parseClientsFile(&inv, *clientsFile)
+		if err != nil {
+			fatal("-client-file:", err)
+		}
+	} else {
+		clientList, err = libhive.ParseClientList(&inv, *clients)
+		if err != nil {
+			fatal("-client:", err)
+		}
+	}
+
+	// Build clients and simulators.
 	if err := runner.Build(ctx, clientList, simList); err != nil {
 		fatal(err)
 	}
@@ -122,6 +141,7 @@ func main() {
 		return
 	}
 
+	// Run simulators.
 	var failCount int
 	for _, sim := range simList {
 		result, err := runner.Run(ctx, sim, env)
@@ -146,10 +166,36 @@ func fatal(args ...interface{}) {
 	os.Exit(1)
 }
 
-func splitAndTrim(input, sep string) []string {
-	list := strings.Split(input, sep)
-	for i := range list {
-		list[i] = strings.TrimSpace(list[i])
+func parseClientsFile(inv *libhive.Inventory, file string) ([]libhive.ClientDesignator, error) {
+	f, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	return libhive.ParseClientListYAML(inv, f)
+}
+
+func checkFlagsExclusive(flagNames ...string) {
+	set := make(map[string]bool)
+	flag.Visit(func(f *flag.Flag) {
+		set[f.Name] = true
+	})
+	var found bool
+	for _, name := range flagNames {
+		if set[name] {
+			if found {
+				fatal("flags", flagListString(flagNames), "cannot be used together")
+			}
+			found = true
+			set[name] = false
+		}
+	}
+}
+
+func flagListString(names []string) string {
+	flags := make([]string, len(names))
+	for i := range flags {
+		flags[i] = "-" + names[i]
 	}
-	return list
+	return strings.Join(flags, ", ")
 }