Merge pull request #2813 from Algo-devops-service/relstable2.10.1

go-algorand 2.10.1-stable

Author: algojohnlee

.circleci/config.yml

@@ -4,17 +4,20 @@ about: Suggestions for how we can improve the algorand platform.
 title: ''
 labels: new-feature-request
 assignees: ''
-
 ---
 
-## Is your feature request related to a problem? Please describe.
+## Problem
+
+<!-- What is the problem that we’re trying to solve? -->
+
+## Solution
 
-<!-- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
+<!-- Do you have a potential/suggested solution? Document more than one if possible. -->
 
-## Describe the solution you'd like.
+## Dependencies
 
-<!-- A clear and concise description of what you want to happen. -->
+<!-- Does the solution have any team or design dependencies? -->
 
-## Additional context.
+## Urgency
 
-<!-- Add any other context or screenshots about the feature request here. -->
+<!-- What is the urgency here and why? -->

.github/ISSUE_TEMPLATE/algorand-engineering-team-issue-template.md

@@ -0,0 +1,90 @@
+name: "ReviewDog workflow"
+on:
+  pull_request:
+jobs:
+  # Blocking Errors Section
+  reviewdog-errors:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # required for new-from-rev option in .golangci.yml
+      - name: Install libraries
+        run: sudo apt-get -y -q install libboost-math-dev
+        # move go out of the way temporarily to avoid "go list ./..." from installing modules
+      - name: Make libsodium.a
+        run: sudo mv /usr/bin/go /usr/bin/go.bak && make crypto/libs/linux/amd64/lib/libsodium.a && sudo mv /usr/bin/go.bak /usr/bin/go
+      - name: reviewdog-golangci-lint
+        uses: reviewdog/action-golangci-lint@v2
+        with:
+          golangci_lint_version: "v1.41.1"
+          golangci_lint_flags: "-c .golangci.yml --allow-parallel-runners"
+          reporter: "github-pr-review"
+          tool_name: "Lint Errors"
+          level: "error"
+          fail_on_error: true
+  # Non-Blocking Warnings Section
+  reviewdog-warnings:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # required for new-from-rev option in .golangci.yml
+      - name: Install libraries
+        run: sudo apt-get -y -q install libboost-math-dev
+      # move go out of the way temporarily to avoid "go list ./..." from installing modules
+      - name: Make libsodium.a
+        run: sudo mv /usr/bin/go /usr/bin/go.bak && make crypto/libs/linux/amd64/lib/libsodium.a && sudo mv /usr/bin/go.bak /usr/bin/go
+      - name: Add bin to PATH
+        run: |
+          echo "$GITHUB_WORKSPACE/bin" >> $GITHUB_PATH
+          echo "$RUNNER_WORKSPACE/$(basename $GITHUB_REPOSITORY)/bin" >> $GITHUB_PATH
+      - name: Install specific golang
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.16.6'
+      - name: Create folders for golangci-lint
+        run: mkdir -p cicdtmp/golangci-lint
+      - name: Check if custom golangci-lint is already built
+        id: cache-golangci-lint
+        uses: actions/cache@v2
+        with:
+          path: cicdtmp/golangci-lint/golangci-lint-cgo
+          key: cicd-golangci-lint-cgo-v0.0.1
+
+      - name: Build custom golangci-lint with CGO_ENABLED
+        if: steps.cache-golangci-lint.outputs.cache-hit != 'true'
+        run: |
+          cd cicdtmp/golangci-lint
+          git clone https://github.com/golangci/golangci-lint.git .
+          git checkout tags/v1.41.1
+          CGO_ENABLED=true go build -trimpath -o golangci-lint-cgo ./cmd/golangci-lint
+          ./golangci-lint-cgo --version
+          cd ../../
+      - name: Install reviewdog
+        run: |
+          curl -sfL https://raw.githubusercontent.com/reviewdog/reviewdog/v0.13.0/install.sh | sh -s
+          reviewdog --version
+      - name: Build custom linters
+        run: |
+          cd cmd/partitiontest_linter/
+          CGO_ENABLED=true go build -buildmode=plugin -trimpath plugin/plugin.go
+          cd ../../
+          ls -la cmd/partitiontest_linter/
+      - name: Run golangci-lint with reviewdog
+        env:
+          REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: >
+          ./cicdtmp/golangci-lint/golangci-lint-cgo run
+          --out-format line-number
+          -c .golangci-warnings.yml
+          --allow-parallel-runners
+          | reviewdog
+          -f=golangci-lint
+          -name="Lint Warnings"
+          -reporter=github-check
+          -filter-mode=added
+          -fail-on-error=false
+          -level=warning

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,76 @@
+run:
+  timeout: 5m
+  tests: true
+  
+linters:
+  disable-all: true
+  enable:
+    - staticcheck
+    - structcheck
+    - typecheck
+    - varcheck
+    - deadcode
+    - gosimple
+    - unused
+    - partitiontest
+
+
+linters-settings:
+  custom:
+    partitiontest:
+      path: cmd/partitiontest_linter/plugin.so
+      description: This custom linter checks files that end in '_test.go', specifically functions that start with 'Test' and have testing argument, for a line 'partitiontest.ParitionTest(<testing arg>)'
+      original-url: github.com/algorand/go-algorand/cmd/partitiontest_linter
+
+severity:
+  default-severity: warning
+
+issues:
+  # use these new lint checks on code since #2574
+  new-from-rev: eb019291beed556ec6ac1ceb4a15114ce4df0c57
+
+  # Disable default exclude rules listed in `golangci-lint run --help` (selectively re-enable some below)
+  exclude-use-default: false
+
+  # Maximum issues count per one linter. Set to 0 to disable. Default is 50.
+  max-issues-per-linter: 0
+
+  # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
+  max-same-issues: 0
+
+  exclude:
+    # ignore govet false positive fixed in https://github.com/golang/go/issues/45043
+    - "sigchanyzer: misuse of unbuffered os.Signal channel as argument to signal.Notify"
+    # ignore golint false positive fixed in https://github.com/golang/lint/pull/487
+    - "exported method (.*).Unwrap` should have comment or be unexported"
+    # ignore issues about the way we use _struct fields to define encoding settings
+    - "`_struct` is unused"
+
+    # Enable some golangci-lint default exception rules:
+    # "EXC0001 errcheck: Almost all programs ignore errors on these functions and in most cases it's ok"
+    - Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked
+    # "EXC0005 staticcheck: Developers tend to write in C-style with an explicit 'break' in a 'switch', so it's ok to ignore"
+    - ineffective break statement. Did you mean to break out of the outer loop
+
+  exclude-rules:
+    # be more lenient with test code
+    - path: _test\.go
+      linters:
+        - staticcheck
+        - structcheck
+        - typecheck
+        - varcheck
+        - deadcode
+        - gosimple
+        - unused
+    # Add all linters here -- Comment this block out for testing linters
+    - path: test/linttest/lintissues\.go
+      linters:
+        - staticcheck
+        - structcheck
+        - typecheck
+        - varcheck
+        - deadcode
+        - gosimple
+        - unused
+        - partitiontest

.github/workflows/reviewdog.yml

@@ -3,35 +3,52 @@ run:
   tests: false
 
 linters:
+  disable-all: true
   enable:
+    - errcheck
+    - gofmt
     - golint
-    - misspell
     - govet
-
-  disable:
-    - deadcode
-    - errcheck
-    - gosimple
     - ineffassign
-    - staticcheck
-    - structcheck
-    - unused
-    - varcheck
-    
-linters-settings:
-#  govet:
-#    check-shadowing: true
+    - misspell
+
+severity:
+  default-severity: error
 
 issues:
-  # don't use default exclude rules listed in `golangci-lint run --help`
+  # use these new lint checks on code since #2574
+  new-from-rev: eb019291beed556ec6ac1ceb4a15114ce4df0c57
+
+  # Disable default exclude rules listed in `golangci-lint run --help` (selectively re-enable some below)
   exclude-use-default: false
 
-  exclude-rules:
+  # Maximum issues count per one linter. Set to 0 to disable. Default is 50.
+  max-issues-per-linter: 0
+
+  # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
+  max-same-issues: 0
+
+  exclude:
     # ignore govet false positive fixed in https://github.com/golang/go/issues/45043
-    - linters:
-        - govet
-      text: "sigchanyzer: misuse of unbuffered os.Signal channel as argument to signal.Notify"
+    - "sigchanyzer: misuse of unbuffered os.Signal channel as argument to signal.Notify"
     # ignore golint false positive fixed in https://github.com/golang/lint/pull/487
-    - linters:
+    - "exported method (.*).Unwrap` should have comment or be unexported"
+    # ignore issues about the way we use _struct fields to define encoding settings
+    - "`_struct` is unused"
+
+    # Enable some golangci-lint default exception rules:
+    # "EXC0001 errcheck: Almost all programs ignore errors on these functions and in most cases it's ok"
+    - Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked
+    # "EXC0005 staticcheck: Developers tend to write in C-style with an explicit 'break' in a 'switch', so it's ok to ignore"
+    - ineffective break statement. Did you mean to break out of the outer loop
+
+  exclude-rules:
+    # Add all linters here -- Comment this block out for testing linters
+    - path: test/linttest/lintissues\.go
+      linters:
+        - errcheck
+        - gofmt
         - golint
-      text: "exported method (.*).Unwrap` should have comment or be unexported"
+        - govet
+        - ineffassign
+        - misspell

.golangci-warnings.yml

@@ -217,8 +217,8 @@ before_cache:
    esac
    docker save -o $HOME/docker_cache/images.tar $(docker images -a -q)
 
-after_success:
-  - scripts/travis/upload_coverage.sh || true
+#after_success:
+#  - scripts/travis/upload_coverage.sh || true
 
 addons:
   apt:

.golangci.yml

@@ -23,6 +23,7 @@ BUILDBRANCH      := $(shell ./scripts/compute_branch.sh)
 CHANNEL          ?= $(shell ./scripts/compute_branch_channel.sh $(BUILDBRANCH))
 DEFAULTNETWORK   ?= $(shell ./scripts/compute_branch_network.sh $(BUILDBRANCH))
 DEFAULT_DEADLOCK ?= $(shell ./scripts/compute_branch_deadlock_default.sh $(BUILDBRANCH))
+export GOCACHE=$(SRCPATH)/tmp/go-cache
 
 GOTAGSLIST          := sqlite_unlock_notify sqlite_omit_load_extension
 
@@ -75,7 +76,7 @@ GOLDFLAGS := $(GOLDFLAGS_BASE) \
 UNIT_TEST_SOURCES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && go list ./... | grep -v /go-algorand/test/ ))
 ALGOD_API_PACKAGES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && cd daemon/algod/api; go list ./... ))
 
-MSGP_GENERATE	:= ./protocol ./crypto ./crypto/compactcert ./data/basics ./data/transactions ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./node ./ledger ./ledger/ledgercore ./compactcert
+MSGP_GENERATE	:= ./protocol ./protocol/test ./crypto ./crypto/compactcert ./data/basics ./data/transactions ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./node ./ledger ./ledger/ledgercore ./compactcert
 
 default: build
 
@@ -203,9 +204,9 @@ build: buildsrc
 # to cache binaries from time to time on empty NFS
 # dirs
 buildsrc: check-go-version crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a node_exporter NONGO_BIN
-	mkdir -p tmp/go-cache && \
-	touch tmp/go-cache/file.txt && \
-	GOCACHE=$(SRCPATH)/tmp/go-cache go install $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./...
+	mkdir -p "${GOCACHE}" && \
+	touch "${GOCACHE}"/file.txt && \
+	go install $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./...
 
 check-go-version:
 	./scripts/check_golang_version.sh build
@@ -321,3 +322,7 @@ include ./scripts/release/mule/Makefile.mule
 
 archive:
 	aws s3 cp tmp/node_pkgs s3://algorand-internal/channel/$(CHANNEL)/$(FULLBUILDNUMBER) --recursive --exclude "*" --include "*$(FULLBUILDNUMBER)*"
+
+build_custom_linters:
+	cd $(SRCPATH)/cmd/partitiontest_linter/ && go build -buildmode=plugin -trimpath plugin/plugin.go && ls plugin.so
+	cd $(SRCPATH)

.travis.yml

@@ -29,10 +29,13 @@ import (
 	"github.com/algorand/go-algorand/data/committee"
 	"github.com/algorand/go-algorand/logging"
 	"github.com/algorand/go-algorand/protocol"
+	"github.com/algorand/go-algorand/test/partitiontest"
 )
 
 // Test Bundle Creation
 func TestBundleCreation(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
 	ledger, addresses, vrfSecrets, otSecrets := readOnlyFixture100()
 	round := ledger.NextRound()
 	period := period(0)
@@ -68,6 +71,8 @@ func TestBundleCreation(t *testing.T) {
 
 // Test Bundle validation with Zero Votes
 func TestBundleCreationWithZeroVotes(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
 	//ledger, addresses, vrfSecrets, otSecrets := readOnlyFixture100()
 	ledger, _, _, _ := readOnlyFixture100()
 
@@ -105,6 +110,8 @@ func makeBundlePanicWrapper(t *testing.T, message string, proposal proposalValue
 
 //Test Bundle Creation with Validation for duplicate votes from same sender
 func TestBundleCreationWithVotesFromSameAddress(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
 	ledger, addresses, vrfSecrets, otSecrets := readOnlyFixture10()
 	round := ledger.NextRound()
 	period := period(0)
@@ -168,6 +175,8 @@ func TestBundleCreationWithVotesFromSameAddress(t *testing.T) {
 
 //Test Bundle Creation with Validation
 func TestBundleCreationWithEquivocationVotes(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
 	ledger, addresses, vrfSecrets, otSecrets := readOnlyFixture10()
 	round := ledger.NextRound()
 	period := period(0)
@@ -276,6 +285,8 @@ func TestBundleCreationWithEquivocationVotes(t *testing.T) {
 
 //Test Bundle Creation with Validation
 func TestBundleCertificationWithEquivocationVotes(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
 	ledger, addresses, vrfSecrets, otSecrets := readOnlyFixture10()
 	round := ledger.NextRound()
 	period := period(0)
@@ -349,6 +360,8 @@ func certificatePanicWrapper(t *testing.T, message string, ub unauthenticatedBun
 
 // Test Bundle Creation with Equivocation Votes under Quorum
 func TestBundleCreationWithEquivocationVotesUnderQuorum(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
 	ledger, addresses, vrfSecrets, otSecrets := readOnlyFixture100()
 	round := ledger.NextRound()
 	period := period(0)