e2e: Add multi-version support to generator (#9933)

* ci: Add missing group to manual E2E workflow

Signed-off-by: Thane Thomson <connect@thanethomson.com>

* e2e: Improve help for multi-version CLI param

Signed-off-by: Thane Thomson <connect@thanethomson.com>

* e2e: Rename multiversion to multiVersion

Signed-off-by: Thane Thomson <connect@thanethomson.com>

* e2e: Extract generator params into struct

Signed-off-by: Thane Thomson <connect@thanethomson.com>

* e2e: Add support for more than 2 versions in multi-version tests

Signed-off-by: Thane Thomson <connect@thanethomson.com>

* e2e: Add generator logic to extract latest release tag from Git repo

Signed-off-by: Thane Thomson <connect@thanethomson.com>

* e2e: Cater for "dev-" prefixes in tags

Signed-off-by: Thane Thomson <connect@thanethomson.com>

* e2e: Add sophisticated weighted version parsing to generator

Signed-off-by: Thane Thomson <connect@thanethomson.com>

* e2e: Add note to E2E readme about multi-version testing

Signed-off-by: Thane Thomson <connect@thanethomson.com>

* e2e: Expand on Docker image comment in readme

Signed-off-by: Thane Thomson <connect@thanethomson.com>

* e2e: Print node version weights when generating testnets

Signed-off-by: Thane Thomson <connect@thanethomson.com>

* ci: Add manual E2E workflow to run multi-version tests

Signed-off-by: Thane Thomson <connect@thanethomson.com>

* e2e: Replace "HEAD" keyword with "local"

Signed-off-by: Thane Thomson <connect@thanethomson.com>

* e2e: Update readme to reflect "HEAD" -> "local" keyword change and expand on examples

Signed-off-by: Thane Thomson <connect@thanethomson.com>

* e2e: Clarify comment relating to Docker image versions

Signed-off-by: Thane Thomson <connect@thanethomson.com>

* Apply suggestions from code review

Co-authored-by: Sergio Mena <sergio@informal.systems>

* e2e: Skip invalid tags instead of erroring in generator

Signed-off-by: Thane Thomson <connect@thanethomson.com>

* e2e: Add comment in version selection code

Signed-off-by: Thane Thomson <connect@thanethomson.com>

* ci: Replace "HEAD" with "local" keyword in multiversion workflow

Signed-off-by: Thane Thomson <connect@thanethomson.com>

* e2e: Use error format specifier when returning wrapped error

Signed-off-by: Thane Thomson <connect@thanethomson.com>

Signed-off-by: Thane Thomson <connect@thanethomson.com>
Co-authored-by: Sergio Mena <sergio@informal.systems>

.github/workflows/e2e-manual-multiversion.yml

@@ -0,0 +1,38 @@
+# Manually run the nightly E2E tests for a particular branch, but test with
+# multiple versions.
+name: e2e-manual-multiversion
+on:
+  workflow_dispatch:
+
+jobs:
+  e2e-manual-multiversion-test:
+    # Run parallel jobs for the listed testnet groups (must match the
+    # ./build/generator -g flag)
+    strategy:
+      fail-fast: false
+      matrix:
+        group: ['00', '01', '02', '03', '04']
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.18'
+
+      - uses: actions/checkout@v3
+
+      - name: Build
+        working-directory: test/e2e
+        # Run make jobs in parallel, since we can't run steps in parallel.
+        run: make -j2 docker generator runner tests
+
+      - name: Generate testnets
+        working-directory: test/e2e
+        # When changing -g, also change the matrix groups above
+        # Generate multi-version tests with double the quantity of E2E nodes
+        # based on the current branch as compared to the latest version.
+        run: ./build/generator -g 5 -m "latest:1,local:2" -d networks/nightly/
+
+      - name: Run ${{ matrix.p2p }} p2p testnets
+        working-directory: test/e2e
+        run: ./run-multiple.sh networks/nightly/*-group${{ matrix.group }}-*.toml

.github/workflows/e2e-manual.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        group: ['00', '01', '02', '03']
+        group: ['00', '01', '02', '03', '04']
     runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:

go.mod

@@ -44,9 +44,11 @@ require (
 )
 
 require (
+	github.com/Masterminds/semver/v3 v3.2.0
 	github.com/btcsuite/btcd/btcec/v2 v2.3.2
 	github.com/btcsuite/btcd/btcutil v1.1.3
 	github.com/cosmos/gogoproto v1.4.3
+	github.com/go-git/go-git/v5 v5.5.1
 	github.com/gofrs/uuid v4.3.1+incompatible
 	github.com/google/uuid v1.3.0
 	github.com/oasisprotocol/curve25519-voi v0.0.0-20220708102147-0a8a51822cae
@@ -69,6 +71,8 @@ require (
 	github.com/Microsoft/go-winio v0.6.0 // indirect
 	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
 	github.com/OpenPeeDeeP/depguard v1.1.1 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect
+	github.com/acomagu/bufpipe v1.0.3 // indirect
 	github.com/alexkohler/prealloc v1.0.0 // indirect
 	github.com/alingse/asasalint v0.0.11 // indirect
 	github.com/ashanbrown/forbidigo v1.3.0 // indirect
@@ -86,6 +90,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/charithe/durationcheck v0.0.9 // indirect
 	github.com/chavacava/garif v0.0.0-20220630083739-93517212f375 // indirect
+	github.com/cloudflare/circl v1.3.1 // indirect
 	github.com/containerd/containerd v1.6.9 // indirect
 	github.com/containerd/continuity v0.3.0 // indirect
 	github.com/containerd/typeurl v1.0.2 // indirect
@@ -103,6 +108,7 @@ require (
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/esimonov/ifshort v1.0.4 // indirect
 	github.com/ettle/strcase v0.1.1 // indirect
 	github.com/fatih/color v1.13.0 // indirect
@@ -113,6 +119,8 @@ require (
 	github.com/fzipp/gocyclo v0.6.0 // indirect
 	github.com/go-chi/chi/v5 v5.0.7 // indirect
 	github.com/go-critic/go-critic v0.6.5 // indirect
+	github.com/go-git/gcfg v1.5.0 // indirect
+	github.com/go-git/go-billy/v5 v5.3.1 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-toolsmith/astcast v1.0.0 // indirect
@@ -151,13 +159,16 @@ require (
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/hexops/gotextdiff v1.0.3 // indirect
+	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jdxcode/netrc v0.0.0-20210204082910-926c7f70242a // indirect
 	github.com/jgautheron/goconst v1.5.1 // indirect
 	github.com/jingyugao/rowserrcheck v1.1.1 // indirect
 	github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af // indirect
 	github.com/jmhodges/levigo v1.0.0 // indirect
 	github.com/julz/importas v0.1.0 // indirect
+	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/kisielk/errcheck v1.6.2 // indirect
 	github.com/kisielk/gotool v1.0.0 // indirect
 	github.com/kkHAIKE/contextcheck v1.1.3 // indirect
@@ -198,6 +209,7 @@ require (
 	github.com/pelletier/go-toml/v2 v2.0.5 // indirect
 	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
 	github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d // indirect
+	github.com/pjbgf/sha1cd v0.2.3 // indirect
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
 	github.com/pkg/profile v1.7.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -218,11 +230,13 @@ require (
 	github.com/sashamelentyev/usestdlibvars v1.20.0 // indirect
 	github.com/satori/go.uuid v1.2.0 // indirect
 	github.com/securego/gosec/v2 v2.13.1 // indirect
+	github.com/sergi/go-diff v1.2.0 // indirect
 	github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/sivchari/containedctx v1.0.2 // indirect
 	github.com/sivchari/nosnakecase v1.7.0 // indirect
 	github.com/sivchari/tenv v1.7.0 // indirect
+	github.com/skeema/knownhosts v1.1.0 // indirect
 	github.com/sonatard/noctx v0.0.1 // indirect
 	github.com/sourcegraph/go-diff v0.6.1 // indirect
 	github.com/spf13/afero v1.9.2 // indirect
@@ -243,6 +257,7 @@ require (
 	github.com/ultraware/funlen v0.0.3 // indirect
 	github.com/ultraware/whitespace v0.0.5 // indirect
 	github.com/uudashr/gocognit v1.0.6 // indirect
+	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/yagipy/maintidx v1.0.0 // indirect
 	github.com/yeya24/promlinter v0.2.0 // indirect
 	gitlab.com/bosi/decorder v0.2.3 // indirect
@@ -261,9 +276,10 @@ require (
 	golang.org/x/sys v0.3.0 // indirect
 	golang.org/x/term v0.3.0 // indirect
 	golang.org/x/text v0.5.0 // indirect
-	golang.org/x/tools v0.3.0 // indirect
+	golang.org/x/tools v0.4.0 // indirect
 	google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	honnef.co/go/tools v0.3.3 // indirect