build: add remote debugging with delve

Makefile

@@ -96,6 +96,11 @@ ifeq (,$(findstring nostrip,$(COSMOS_BUILD_OPTIONS)))
   BUILD_FLAGS += -trimpath
 endif
 
+# Check for debug option
+ifeq (debug,$(findstring debug,$(COSMOS_BUILD_OPTIONS)))
+  BUILD_FLAGS += -gcflags "all=-N -l"
+endif
+
 all: tools build lint test
 
 # The below include contains the tools and runsim targets.
@@ -472,17 +477,28 @@ proto-update-deps:
 ###                                Localnet                                 ###
 ###############################################################################
 
-# Run a 4-node testnet locally via docker compose
-localnet-start: localnet-stop
-	$(if $(shell $(DOCKER) inspect -f '{{ .Id }}' cosmossdk/simd-env 2>/dev/null),$(info found image cosmossdk/simd-env),$(MAKE) -C contrib/images simd-env)
-	$(DOCKER) run --rm -v $(CURDIR)/localnet:/data cosmossdk/simd-env \
-		testnet init-files --v 4 -o /data --starting-ip-address 192.168.10.2 --keyring-backend=test
+localnet-build-env:
+	$(MAKE) -C contrib/images simd-env
+localnet-build-dlv:
+	$(MAKE) -C contrib/images simd-dlv
+
+localnet-build-nodes:
+	$(DOCKER) run --rm -v $(CURDIR)/.testnets:/data cosmossdk/simd \
+    		  testnet init-files --v 4 --starting-ip-address 192.168.10.2 --keyring-backend=test
 	docker-compose up -d
 
 localnet-stop:
 	docker-compose down
 
-.PHONY: localnet-start localnet-stop
+# localnet-start will run a 4-node testnet locally. The nodes are
+# based off the docker images in: ./contrib/images/simd-env
+localnet-start: localnet-stop localnet-build-env localnet-build-nodes
+
+# localnet-debug will run a 4-node testnet locally in debug mode
+# you can read more about the debug mode here: ./contrib/images/simd-dlv/README.md
+localnet-debug: localnet-stop localnet-build-dlv localnet-build-nodes
+
+.PHONY: localnet-start localnet-stop localnet-debug localnet-build-env localnet-build-dlv localnet-build-nodes
 
 ###############################################################################
 ###                                rosetta                                  ###

contrib/images/Makefile

@@ -1,7 +1,14 @@
 all: simd-env
 
-simd-env:
-	docker build --tag cosmossdk/simd-env -f simd-env/Dockerfile \
-		$(shell git rev-parse --show-toplevel)
+simd-env: simd-rmi
+	docker build --tag cosmossdk/simd -f simd-env/Dockerfile \
+    		$(shell git rev-parse --show-toplevel)
 
-.PHONY: all simd-env
+simd-dlv: simd-rmi
+	docker build --tag cosmossdk/simd -f simd-dlv/Dockerfile \
+        	$(shell git rev-parse --show-toplevel)
+
+simd-rmi:
+	docker rmi cosmossdk/simd 2>/dev/null; true
+
+.PHONY: all simd-env simd-dlv simd-rmi

contrib/images/simd-dlv/Dockerfile

@@ -0,0 +1,23 @@
+FROM golang:1.17-alpine AS build
+RUN apk add build-base git linux-headers libc-dev
+RUN go install github.com/go-delve/delve/cmd/dlv@latest
+WORKDIR /work
+COPY go.mod go.sum /work/
+COPY db/go.mod db/go.sum /work/db/
+RUN go mod download
+COPY ./ /work
+RUN LEDGER_ENABLED=false make COSMOS_BUILD_OPTIONS="debug,nostrip" clean build
+
+FROM alpine:3.14 AS run
+RUN apk add bash curl jq
+COPY contrib/images/simd-dlv/wrapper.sh /usr/bin/wrapper.sh
+
+VOLUME /simd
+COPY --from=build /work/build/simd /simd/
+COPY --from=build /go/bin/dlv /usr/local/bin
+WORKDIR /simd
+
+EXPOSE 26656 26657 2345
+ENTRYPOINT ["/usr/bin/wrapper.sh"]
+CMD ["start", "--log_format", "plain"]
+STOPSIGNAL SIGTERM

contrib/images/simd-dlv/README.md

@@ -0,0 +1,34 @@
+# Remote Debugging with go-delve
+
+[Delve](https://github.com/go-delve/delve) is a debugger for the Go programming language. The goal of the project is to provide a simple, full featured debugging tool for Go. Delve should be easy to invoke and easy to use. Chances are if you're using a debugger, things aren't going your way. With that in mind, Delve should stay out of your way as much as possible.
+
+## Use-case
+
+Cosmos-SDK provides you with a local network to bootstrap a chain in your machine, but how does one debug a node or module?
+
+If we start a single node, we won't be able to debug transactions as the machine will be in bootstrapping phase trying to find peers to connect too, thats why we need to start a local network.
+
+But the current `localnet-start` does not provide us with debugging tools so that's why there is a different image for debugging a local network, that is to avoid any issues in the future were debugging won't be needed.
+
+Both `simd-env` and `simd-dlv` work and run the same, except that `simd-dlv` uses `go-delve` to run the binaries.
+
+## How to use
+
+The command to start a local network in debug mode is:
+```shell
+# make localnet-debug
+```
+
+The command to stop the local network and destroy its containers is:
+```shell
+# make localnet-stop
+```
+__note: this works the same for both `localnet-start` and `localnet-debug`__
+
+Now, by default only `simdnode0` is run in debug mode, but you can run any of the other nodes in debug mode by changing the `DEBUG` environment variable to `1` in `docker-compose.yml`.
+
+## How to connect
+
+Delve will open a port on `2345` for `simdnode0` and it will increment for each of the other nodes that have `DEBUG` set to `1`.
+
+You can connect to the debugging server either through [GoLand IDE](https://www.jetbrains.com/help/go/attach-to-running-go-processes-with-debugger.html)  or you can use  [delve cli](https://github.com/go-delve/delve/blob/master/Documentation/usage/dlv_connect.md) command.

contrib/images/simd-dlv/wrapper.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env sh
+set -euo pipefail
+set -x
+
+DEBUG=${DEBUG:-0}
+BINARY=/simd/${BINARY:-simd}
+ID=${ID:-0}
+LOG=${LOG:-simd.log}
+
+if ! [ -f "${BINARY}" ]; then
+	echo "The binary $(basename "${BINARY}") cannot be found. Please add the binary to the shared folder. Please use the BINARY environment variable if the name of the binary is not 'simd'"
+	exit 1
+fi
+
+export SIMDHOME="/data/node${ID}/simd"
+
+if [ "$DEBUG" -eq 1 ]; then
+  dlv --listen=:2345 --continue --headless=true --api-version=2 --accept-multiclient exec "${BINARY}" -- --home "${SIMDHOME}" "$@"
+elif [ "$DEBUG" -eq 1 ] && [ -d "$(dirname "${SIMDHOME}"/"${LOG}")" ]; then
+  dlv --listen=:2345 --continue --headless=true --api-version=2 --accept-multiclient exec "${BINARY}" -- --home "${SIMDHOME}" "$@" | tee "${SIMDHOME}/${LOG}"
+elif [ -d "$(dirname "${SIMDHOME}"/"${LOG}")" ]; then
+  "${BINARY}" --home "${SIMDHOME}" "$@" | tee "${SIMDHOME}/${LOG}"
+else
+  "${BINARY}" --home "${SIMDHOME}" "$@"
+fi

docker-compose.yml

@@ -3,64 +3,88 @@ version: "3"
 services:
   simdnode0:
     container_name: simdnode0
-    image: "cosmossdk/simd-env"
+    image: "cosmossdk/simd"
+    environment:
+      - DEBUG=1
+      - ID=0
+      - LOG=${LOG:-simd.log}
+    cap_add:
+      - SYS_PTRACE
+    security_opt:
+      - seccomp:unconfined
     ports:
       - "26656-26657:26656-26657"
       - "1317:1317"
       - "9090:9090"
-    environment:
-      - ID=0
-      - LOG=${LOG:-simd.log}
+      - "2345:2345"
     volumes:
-      - ./localnet:/data:Z
+      - ./.testnets:/data:Z
     networks:
       localnet:
         ipv4_address: 192.168.10.2
 
   simdnode1:
     container_name: simdnode1
-    image: "cosmossdk/simd-env"
+    image: "cosmossdk/simd"
+    environment:
+      - DEBUG=0
+      - ID=1
+      - LOG=${LOG:-simd.log}
+    cap_add:
+      - SYS_PTRACE
+    security_opt:
+      - seccomp:unconfined
     ports:
       - "26666-26667:26656-26657"
       - "1318:1317"
       - "9091:9090"
-    environment:
-      - ID=1
-      - LOG=${LOG:-simd.log}
+      - "2346:2345"
     volumes:
-      - ./localnet:/data:Z
+      - ./.testnets:/data:Z
     networks:
       localnet:
         ipv4_address: 192.168.10.3
 
   simdnode2:
     container_name: simdnode2
-    image: "cosmossdk/simd-env"
+    image: "cosmossdk/simd"
     environment:
+      - DEBUG=0
       - ID=2
       - LOG=${LOG:-simd.log}
+    cap_add:
+      - SYS_PTRACE
+    security_opt:
+      - seccomp:unconfined
     ports:
       - "26676-26677:26656-26657"
       - "1319:1317"
       - "9092:9090"
+      - "2347:2345"
     volumes:
-      - ./localnet:/data:Z
+      - ./.testnets:/data:Z
     networks:
       localnet:
         ipv4_address: 192.168.10.4
 
   simdnode3:
     container_name: simdnode3
-    image: "cosmossdk/simd-env"
+    image: "cosmossdk/simd"
     environment:
+      - DEBUG=0
       - ID=3
       - LOG=${LOG:-simd.log}
+    cap_add:
+      - SYS_PTRACE
+    security_opt:
+      - seccomp:unconfined
     ports:
       - "26686-26687:26656-26657"
       - "1320:1317"
       - "9093:9090"
+      - "2348:2345"
     volumes:
-      - ./localnet:/data:Z
+      - ./.testnets:/data:Z
     networks:
       localnet:
         ipv4_address: 192.168.10.5