fix all XDC&ETH tests and enable it in CI

Author: wjrjerome

.travis.yml

@@ -15,6 +15,15 @@ jobs:
       script:
         - go run build/ci.go lint
 
+    - stage: build
+      os: linux
+      dist: bionic
+      go: 1.15.x
+      env:
+        - GO111MODULE=auto
+      script:
+        - go run build/ci.go test -coverage $TEST_PACKAGES
+
     # These builders run the tests
     # - stage: build
     #   os: linux
@@ -24,13 +33,4 @@ jobs:
     #   env:
     #     - GO111MODULE=auto
     #   script:
-    #     - go run build/ci.go test -coverage $TEST_PACKAGES
-
-    # - stage: build
-    #   os: linux
-    #   dist: bionic
-    #   go: 1.15.x
-    #   env:
-    #     - GO111MODULE=auto
-    #   script:
     #     - go run build/ci.go test -coverage $TEST_PACKAGES

accounts/keystore/plain_test.go

@@ -245,14 +245,6 @@ func loadKeyStoreTestV1(file string, t *testing.T) map[string]KeyStoreTestV1 {
 	return tests
 }
 
-func TestKeyForDirectICAP(t *testing.T) {
-	t.Parallel()
-	key := NewKeyForDirectICAP(rand.Reader)
-	if !strings.HasPrefix(key.Address.Hex(), "0x00") {
-		t.Errorf("Expected first address byte to be zero, have: %s", key.Address.Hex())
-	}
-}
-
 func TestV3_31_Byte_Key(t *testing.T) {
 	t.Parallel()
 	tests := loadKeyStoreTestV3("testdata/v3_test_vector.json", t)

build/ci.go

@@ -18,19 +18,14 @@
 
 /*
 The ci command is called from Continuous Integration scripts.
-
 Usage: go run build/ci.go <command> <command flags/arguments>
-
 Available commands are:
-
    install    [ -arch architecture ] [ -cc compiler ] [ packages... ]                          -- builds packages and executables
    test       [ -coverage ] [ packages... ]                                                    -- runs the tests
    lint                                                                                        -- runs certain pre-selected linters
    importkeys                                                                                  -- imports signing keys from env
    xgo        [ -alltools ] [ options ]                                                        -- cross builds according to options
-
 For all commands, -n prevents execution of external programs (dry run mode).
-
 */
 package main
 
@@ -60,11 +55,11 @@ var (
 		executablePath("geth"),
 		executablePath("puppeth"),
 		executablePath("rlpdump"),
+		executablePath("swarm"),
 		executablePath("wnode"),
 	}
 
-	// Packages to be cross-compiled by the xgo command
-	allCrossCompiledArchiveFiles = allToolsArchiveFiles
+	dlgoVersion = "1.16.4"
 )
 
 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@@ -214,27 +209,39 @@ func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd
 // Running The Tests
 //
 // "tests" also includes static analysis tools such as vet.
-
 func doTest(cmdline []string) {
-	coverage := flag.Bool("coverage", false, "Whether to record code coverage")
+	var (
+		dlgo     = flag.Bool("dlgo", false, "Download Go and build with it")
+		arch     = flag.String("arch", "", "Run tests for given architecture")
+		cc       = flag.String("cc", "", "Sets C compiler binary")
+		coverage = flag.Bool("coverage", false, "Whether to record code coverage")
+		verbose  = flag.Bool("v", false, "Whether to log verbosely")
+	)
 	flag.CommandLine.Parse(cmdline)
-	env := build.Env()
-
-	packages := []string{"./..."}
-	if len(flag.CommandLine.Args()) > 0 {
-		packages = flag.CommandLine.Args()
+	fmt.Printf("Running tests with command line %v \n", cmdline)
+	// Configure the toolchain.
+	tc := build.GoToolchain{GOARCH: *arch, CC: *cc}
+	if *dlgo {
+		csdb := build.MustLoadChecksums("build/checksums.txt")
+		tc.Root = build.DownloadGo(csdb, dlgoVersion)
 	}
-	packages = build.ExpandPackagesNoVendor(packages)
+	gotest := tc.Go("test")
 
-	// Run the actual tests.
 	// Test a single package at a time. CI builders are slow
 	// and some tests run into timeouts under load.
-	gotest := goTool("test", buildFlags(env)...)
-	gotest.Args = append(gotest.Args, "-p", "1", "-timeout", "5m")
+	gotest.Args = append(gotest.Args, "-p", "1")
 	if *coverage {
 		gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover")
 	}
+	if *verbose {
+		gotest.Args = append(gotest.Args, "-v")
+	}
 
+	packages := []string{"./..."}
+	if len(flag.CommandLine.Args()) > 0 {
+		packages = flag.CommandLine.Args()
+	}
+	fmt.Printf("Running tests for %v \n", packages)
 	gotest.Args = append(gotest.Args, packages...)
 	build.MustRun(gotest)
 }
@@ -245,7 +252,6 @@ func doLint(cmdline []string) {
 		cachedir = flag.String("cachedir", "./build/cache", "directory for caching golangci-lint binary.")
 	)
 	flag.CommandLine.Parse(cmdline)
-
 	packages := []string{"./..."}
 	if len(flag.CommandLine.Args()) > 0 {
 		packages = flag.CommandLine.Args()
@@ -255,7 +261,6 @@ func doLint(cmdline []string) {
 	lflags := []string{"run", "--config", ".golangci.yml"}
 	build.MustRunCommand(linter, append(lflags, packages...)...)
 	fmt.Println("You have achieved perfection.")
-
 }
 
 // downloadLinter downloads and unpacks golangci-lint.
@@ -293,7 +298,7 @@ func doXgo(cmdline []string) {
 
 	if *alltools {
 		args = append(args, []string{"--dest", GOBIN}...)
-		for _, res := range allCrossCompiledArchiveFiles {
+		for _, res := range allToolsArchiveFiles {
 			if strings.HasPrefix(res, GOBIN) {
 				// Binary tool found, cross build it explicitly
 				args = append(args, "./"+filepath.Join("cmd", filepath.Base(res)))

crypto/ecies/ecies_test.go

@@ -47,7 +47,7 @@ var dumpEnc bool
 
 func init() {
 	flDump := flag.Bool("dump", false, "write encrypted test message to file")
-	flag.Parse()
+	// flag.Parse()
 	dumpEnc = *flDump
 }
 

internal/build/gotool.go

@@ -0,0 +1,149 @@
+// Copyright 2021 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package build
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+)
+
+type GoToolchain struct {
+	Root string // GOROOT
+
+	// Cross-compilation variables. These are set when running the go tool.
+	GOARCH string
+	GOOS   string
+	CC     string
+}
+
+// Go creates an invocation of the go command.
+func (g *GoToolchain) Go(command string, args ...string) *exec.Cmd {
+	tool := g.goTool(command, args...)
+
+	// Configure environment for cross build.
+	if g.GOARCH != "" && g.GOARCH != runtime.GOARCH {
+		tool.Env = append(tool.Env, "CGO_ENABLED=1")
+		tool.Env = append(tool.Env, "GOARCH="+g.GOARCH)
+	}
+	if g.GOOS != "" && g.GOOS != runtime.GOOS {
+		tool.Env = append(tool.Env, "GOOS="+g.GOOS)
+	}
+	// Configure C compiler.
+	if g.CC != "" {
+		tool.Env = append(tool.Env, "CC="+g.CC)
+	} else if os.Getenv("CC") != "" {
+		tool.Env = append(tool.Env, "CC="+os.Getenv("CC"))
+	}
+	return tool
+}
+
+// Install creates an invocation of 'go install'. The command is configured to output
+// executables to the given 'gobin' directory.
+//
+// This can be used to install auxiliary build tools without modifying the local go.mod and
+// go.sum files. To install tools which are not required by go.mod, ensure that all module
+// paths in 'args' contain a module version suffix (e.g. "...@latest").
+func (g *GoToolchain) Install(gobin string, args ...string) *exec.Cmd {
+	if !filepath.IsAbs(gobin) {
+		panic("GOBIN must be an absolute path")
+	}
+	tool := g.goTool("install")
+	tool.Env = append(tool.Env, "GOBIN="+gobin)
+	tool.Args = append(tool.Args, "-mod=readonly")
+	tool.Args = append(tool.Args, args...)
+
+	// Ensure GOPATH is set because go install seems to absolutely require it. This uses
+	// 'go env' because it resolves the default value when GOPATH is not set in the
+	// environment. Ignore errors running go env and leave any complaining about GOPATH to
+	// the install command.
+	pathTool := g.goTool("env", "GOPATH")
+	output, _ := pathTool.Output()
+	tool.Env = append(tool.Env, "GOPATH="+string(output))
+	return tool
+}
+
+func (g *GoToolchain) goTool(command string, args ...string) *exec.Cmd {
+	if g.Root == "" {
+		g.Root = runtime.GOROOT()
+	}
+	tool := exec.Command(filepath.Join(g.Root, "bin", "go"), command)
+	tool.Args = append(tool.Args, args...)
+	tool.Env = append(tool.Env, "GOROOT="+g.Root)
+
+	// Forward environment variables to the tool, but skip compiler target settings.
+	// TODO: what about GOARM?
+	skip := map[string]struct{}{"GOROOT": {}, "GOARCH": {}, "GOOS": {}, "GOBIN": {}, "CC": {}}
+	for _, e := range os.Environ() {
+		if i := strings.IndexByte(e, '='); i >= 0 {
+			if _, ok := skip[e[:i]]; ok {
+				continue
+			}
+		}
+		tool.Env = append(tool.Env, e)
+	}
+	return tool
+}
+
+// DownloadGo downloads the Go binary distribution and unpacks it into a temporary
+// directory. It returns the GOROOT of the unpacked toolchain.
+func DownloadGo(csdb *ChecksumDB, version string) string {
+	// Shortcut: if the Go version that runs this script matches the
+	// requested version exactly, there is no need to download anything.
+	activeGo := strings.TrimPrefix(runtime.Version(), "go")
+	if activeGo == version {
+		log.Printf("-dlgo version matches active Go version %s, skipping download.", activeGo)
+		return runtime.GOROOT()
+	}
+
+	ucache, err := os.UserCacheDir()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// For Arm architecture, GOARCH includes ISA version.
+	os := runtime.GOOS
+	arch := runtime.GOARCH
+	if arch == "arm" {
+		arch = "armv6l"
+	}
+	file := fmt.Sprintf("go%s.%s-%s", version, os, arch)
+	if os == "windows" {
+		file += ".zip"
+	} else {
+		file += ".tar.gz"
+	}
+	url := "https://golang.org/dl/" + file
+	dst := filepath.Join(ucache, file)
+	if err := csdb.DownloadFile(url, dst); err != nil {
+		log.Fatal(err)
+	}
+
+	godir := filepath.Join(ucache, fmt.Sprintf("geth-go-%s-%s-%s", version, os, arch))
+	if err := ExtractArchive(dst, godir); err != nil {
+		log.Fatal(err)
+	}
+	goroot, err := filepath.Abs(filepath.Join(godir, "go"))
+	if err != nil {
+		log.Fatal(err)
+	}
+	return goroot
+}