From 916124b9eaf723a280cd721231f8dfd92bedd942 Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Mon, 8 Apr 2019 22:58:17 -0400 Subject: [PATCH] Adding `make debug` support to build debug binaries and debug containers which wrap the binary with delve and allow for remote debugging --- .gitignore | 4 +++ Makefile | 50 ++++++++++++++++++++++++++++-- cmd/loki/Dockerfile.debug | 17 +++++++++++ cmd/promtail/Dockerfile.debug | 17 +++++++++++ debug/README.md | 57 +++++++++++++++++++++++++++++++++++ debug/docker-compose.yaml | 37 +++++++++++++++++++++++ loki-build-image/Dockerfile | 1 + 7 files changed, 180 insertions(+), 3 deletions(-) create mode 100644 cmd/loki/Dockerfile.debug create mode 100644 cmd/promtail/Dockerfile.debug create mode 100644 debug/README.md create mode 100644 debug/docker-compose.yaml diff --git a/.gitignore b/.gitignore index dec47fd3a960..2c6409bbd5cd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .uptodate +.uptodate-debug .pkg .cache *.output @@ -7,6 +8,9 @@ requirements.lock mixin/vendor/ cmd/loki/loki cmd/promtail/promtail +cmd/loki/loki-debug +cmd/promtail/promtail-debug /loki /promtail /logcli +dlv \ No newline at end of file diff --git a/Makefile b/Makefile index e7c27e0f4189..7c91bf63bfb0 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,7 @@ CHARTS := production/helm/loki production/helm/promtail production/helm/loki-sta IMAGE_PREFIX ?= grafana/ IMAGE_TAG := $(shell ./tools/image-tag) UPTODATE := .uptodate +DEBUG_UPTODATE := .uptodate-debug GIT_REVISION := $(shell git rev-parse --short HEAD) GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) @@ -21,6 +22,11 @@ GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) $(SUDO) docker tag $(IMAGE_PREFIX)$(shell basename $(@D)) $(IMAGE_PREFIX)$(shell basename $(@D)):$(IMAGE_TAG) touch $@ +%/$(DEBUG_UPTODATE): %/Dockerfile.debug + $(SUDO) docker build -f $(@D)/Dockerfile.debug -t $(IMAGE_PREFIX)$(shell basename $(@D))-debug $(@D)/ + $(SUDO) docker tag $(IMAGE_PREFIX)$(shell basename $(@D))-debug $(IMAGE_PREFIX)$(shell basename $(@D))-debug:$(IMAGE_TAG) + touch $@ + # We don't want find to scan inside a bunch of directories, to accelerate the # 'make: Entering directory '/go/src/github.com/grafana/loki' phase. DONT_FIND := -name tools -prune -o -name vendor -prune -o -name .git -prune -o -name .cache -prune -o -name .pkg -prune -o @@ -28,8 +34,13 @@ DONT_FIND := -name tools -prune -o -name vendor -prune -o -name .git -prune -o - # Get a list of directories containing Dockerfiles DOCKERFILES := $(shell find . $(DONT_FIND) -type f -name 'Dockerfile' -print) UPTODATE_FILES := $(patsubst %/Dockerfile,%/$(UPTODATE),$(DOCKERFILES)) +DEBUG_DOCKERFILES := $(shell find . $(DONT_FIND) -type f -name 'Dockerfile.debug' -print) +DEBUG_UPTODATE_FILES := $(patsubst %/Dockerfile.debug,%/$(DEBUG_UPTODATE),$(DEBUG_DOCKERFILES)) +DEBUG_DLV_FILES := $(patsubst %/Dockerfile.debug,%/dlv,$(DEBUG_DOCKERFILES)) DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile,%,$(DOCKERFILES)) IMAGE_NAMES := $(foreach dir,$(DOCKER_IMAGE_DIRS),$(patsubst %,$(IMAGE_PREFIX)%,$(shell basename $(dir)))) +DEBUG_DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile.debug,%,$(DEBUG_DOCKERFILES)) +DEBUG_IMAGE_NAMES := $(foreach dir,$(DEBUG_DOCKER_IMAGE_DIRS),$(patsubst %,$(IMAGE_PREFIX)%,$(shell basename $(dir))-debug)) images: $(info $(patsubst %,%:$(IMAGE_TAG),$(IMAGE_NAMES))) @echo > /dev/null @@ -49,13 +60,29 @@ YACC_GOS := $(patsubst %.y,%.go,$(YACC_DEFS)) # for every directory with main.go in it, in the ./cmd directory. MAIN_GO := $(shell find . $(DONT_FIND) -type f -name 'main.go' -print) EXES := $(foreach exe, $(patsubst ./cmd/%/main.go, %, $(MAIN_GO)), ./cmd/$(exe)/$(exe)) +DEBUG_EXES := $(foreach exe, $(patsubst ./cmd/%/main.go, %, $(MAIN_GO)), ./cmd/$(exe)/$(exe)-debug) GO_FILES := $(shell find . $(DONT_FIND) -name cmd -prune -o -type f -name '*.go' -print) + +# This is the important part of how `make all` enters this file +# the above EXES finds all the main.go files and for each of them +# it creates the dep_exe targets which look like this: +# cmd/promtail/promtail: loki-build-image/.uptodate cmd/promtail//main.go pkg/loki/loki.go pkg/loki/fake_auth.go ... +# cmd/promtail/.uptodate: cmd/promtail/promtail +# Then when `make all` expands `$(UPTODATE_FILES)` it will call the second generated target and initiate the build process define dep_exe $(1): $(dir $(1))/main.go $(GO_FILES) $(PROTO_GOS) $(YACC_GOS) $(dir $(1))$(UPTODATE): $(1) endef $(foreach exe, $(EXES), $(eval $(call dep_exe, $(exe)))) +# Everything is basically duplicated for debug builds, +# but with a different set of Dockerfiles and binaries appended with -debug. +define debug_dep_exe +$(1): $(dir $(1))/main.go $(GO_FILES) $(PROTO_GOS) $(YACC_GOS) +$(dir $(1))$(DEBUG_UPTODATE): $(1) +endef +$(foreach exe, $(DEBUG_EXES), $(eval $(call debug_dep_exe, $(exe)))) + # Manually declared dependancies and what goes into each exe pkg/logproto/logproto.pb.go: pkg/logproto/logproto.proto vendor/github.com/cortexproject/cortex/pkg/ring/ring.pb.go: vendor/github.com/cortexproject/cortex/pkg/ring/ring.proto @@ -65,6 +92,7 @@ pkg/parser/labels.go: pkg/parser/labels.y pkg/parser/matchers.go: pkg/parser/matchers.y all: $(UPTODATE_FILES) test: $(PROTO_GOS) $(YACC_GOS) +debug: $(DEBUG_UPTODATE_FILES) yacc: $(YACC_GOS) protos: $(PROTO_GOS) yacc: $(YACC_GOS) @@ -86,6 +114,10 @@ TTY := --tty VPREFIX := github.com/grafana/loki/vendor/github.com/prometheus/common/version GO_FLAGS := -ldflags "-extldflags \"-static\" -s -w -X $(VPREFIX).Branch=$(GIT_BRANCH) -X $(VPREFIX).Version=$(IMAGE_TAG) -X $(VPREFIX).Revision=$(GIT_REVISION)" -tags netgo +# Per some websites I've seen to add `-gcflags "all=-N -l"`, the gcflags seem poorly if at all documented +# the best I could dig up is -N disables optimizations and -l disables inlining which should make debugging match source better. +# Also remove the -s and -w flags present in the normal build which strip the symbol table and the DWARF symbol table. +DEBUG_GO_FLAGS := -gcflags "all=-N -l" -ldflags "-extldflags \"-static\" -X $(VPREFIX).Branch=$(GIT_BRANCH) -X $(VPREFIX).Version=$(IMAGE_TAG) -X $(VPREFIX).Revision=$(GIT_REVISION)" -tags netgo NETGO_CHECK = @strings $@ | grep cgo_stub\\\.go >/dev/null || { \ rm $@; \ @@ -96,9 +128,15 @@ NETGO_CHECK = @strings $@ | grep cgo_stub\\\.go >/dev/null || { \ false; \ } +# If BUILD_IN_CONTAINER is true, the build image is run which launches +# an image that mounts this project as a volume. The image invokes a build.sh script +# which essentially re-enters this file with BUILD_IN_CONTAINER=FALSE +# causing the else block target below to be called and the files to be built. +# If BUILD_IN_CONTAINER were false to begin with, the else block is +# executed and the binaries are built without ever launching the build container. ifeq ($(BUILD_IN_CONTAINER),true) -$(EXES) $(PROTO_GOS) $(YACC_GOS) lint test shell check-generated-files: loki-build-image/$(UPTODATE) +$(EXES) $(DEBUG_EXES) $(PROTO_GOS) $(YACC_GOS) lint test shell check-generated-files: loki-build-image/$(UPTODATE) @mkdir -p $(shell pwd)/.pkg @mkdir -p $(shell pwd)/.cache $(SUDO) docker run $(RM) $(TTY) -i \ @@ -109,6 +147,12 @@ $(EXES) $(PROTO_GOS) $(YACC_GOS) lint test shell check-generated-files: loki-bui else +$(DEBUG_EXES): loki-build-image/$(UPTODATE) + CGO_ENABLED=0 go build $(DEBUG_GO_FLAGS) -o $@ ./$(@D) + $(NETGO_CHECK) + # Copy the delve binary to make it easily available to put in the binary's container. + [ -f "/go/bin/dlv" ] && mv "/go/bin/dlv" $(@D)/dlv + $(EXES): loki-build-image/$(UPTODATE) CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D) $(NETGO_CHECK) @@ -200,6 +244,6 @@ helm-publish: helm git push origin gh-pages clean: - $(SUDO) docker rmi $(IMAGE_NAMES) >/dev/null 2>&1 || true - rm -rf $(UPTODATE_FILES) $(EXES) .cache + $(SUDO) docker rmi $(IMAGE_NAMES) $(DEBUG_IMAGE_NAMES) >/dev/null 2>&1 || true + rm -rf $(UPTODATE_FILES) $(EXES) $(DEBUG_UPTODATE_FILES) $(DEBUG_EXES) $(DEBUG_DLV_FILES) .cache go clean ./... diff --git a/cmd/loki/Dockerfile.debug b/cmd/loki/Dockerfile.debug new file mode 100644 index 000000000000..ca4c61f86eec --- /dev/null +++ b/cmd/loki/Dockerfile.debug @@ -0,0 +1,17 @@ +FROM alpine:3.9 +RUN apk add --update --no-cache ca-certificates +COPY loki-debug /bin/loki-debug +ADD dlv /usr/bin +COPY loki-local-config.yaml /etc/loki/local-config.yaml +EXPOSE 80 + +# Expose 40000 for delve +EXPOSE 40000 + +# Allow delve to run on Alpine based containers. +RUN apk add --no-cache libc6-compat + +# Run delve, ending with -- because we pass params via kubernetes, per the docs: +# Pass flags to the program you are debugging using --, for example:` +# dlv exec ./hello -- server --config conf/config.toml` +ENTRYPOINT ["/usr/bin/dlv", "--listen=:40000", "--headless=true", "--api-version=2", "exec", "/bin/loki-debug", "--"] diff --git a/cmd/promtail/Dockerfile.debug b/cmd/promtail/Dockerfile.debug new file mode 100644 index 000000000000..48a0a2113b5d --- /dev/null +++ b/cmd/promtail/Dockerfile.debug @@ -0,0 +1,17 @@ +FROM alpine:3.9 +RUN apk add --update --no-cache ca-certificates +ADD promtail-debug /usr/bin +ADD dlv /usr/bin +COPY promtail-local-config.yaml /etc/promtail/local-config.yaml +COPY promtail-docker-config.yaml /etc/promtail/docker-config.yaml + +# Expose 40000 for delve +EXPOSE 40000 + +# Allow delve to run on Alpine based containers. +RUN apk add --no-cache libc6-compat + +# Run delve, ending with -- because we pass params via kubernetes, per the docs: +# Pass flags to the program you are debugging using --, for example:` +# dlv exec ./hello -- server --config conf/config.toml` +ENTRYPOINT ["/usr/bin/dlv", "--listen=:40000", "--headless=true", "--api-version=2", "exec", "/usr/bin/promtail-debug", "--"] \ No newline at end of file diff --git a/debug/README.md b/debug/README.md new file mode 100644 index 000000000000..b4e571325a2a --- /dev/null +++ b/debug/README.md @@ -0,0 +1,57 @@ +## Debug images + +To build debug images run + +```shell +make debug +``` + +You can use the `docker-compose.yaml` in this directory to launch the debug versions of the image in docker + + +## Promtail in kubernetes + +If you want to debug promtail in kubernetes, I have done so with the ksonnet setup: + +```shell +ks init promtail +cd promtail +ks env add promtail +jb init +jb install github.com/grafana/loki/production/ksonnet/promtail +vi environments/promtail/main.jsonnet +``` + +Replace the contents with: + +```jsonnet +local promtail = import 'promtail/promtail.libsonnet'; + + +promtail + { + _images+:: { + promtail: 'grafana/promtail-debug:latest', + }, + _config+:: { + namespace: 'default', + + promtail_config+: { + external_labels+: { + cluster: 'some_cluster_name', + }, + scheme: 'https', + hostname: 'hostname', + username: 'username', + password: 'password', + }, + }, +} +``` + +change the `some_cluster_name` to anything meaningful to help find your logs in loki + +also update the `hostname`, `username`, and `password` for your loki instance. + +## Loki in kubernetes + +Haven't tried this yet, it works from docker-compose so it should run in kubernetes just fine also. \ No newline at end of file diff --git a/debug/docker-compose.yaml b/debug/docker-compose.yaml new file mode 100644 index 000000000000..88c0811ac125 --- /dev/null +++ b/debug/docker-compose.yaml @@ -0,0 +1,37 @@ +version: "3" + +networks: + loki: + +services: + loki: + # this is required according to https://github.com/Microsoft/vscode-go/wiki/Debugging-Go-code-using-VS-Code#linuxdocker + security_opt: + - seccomp:unconfined + image: grafana/loki-debug:latest + ports: + - "40000:40000" + - "3100:3100" + command: -config.file=/etc/loki/local-config.yaml + networks: + - loki + + promtail: + # this is required according to https://github.com/Microsoft/vscode-go/wiki/Debugging-Go-code-using-VS-Code#linuxdocker + security_opt: + - seccomp:unconfined + image: grafana/promtail-debug:latest + ports: + - "40100:40000" + volumes: + - /var/log:/var/log + command: -config.file=/etc/promtail/docker-config.yaml + networks: + - loki + + grafana: + image: grafana/grafana:master + ports: + - "3000:3000" + networks: + - loki diff --git a/loki-build-image/Dockerfile b/loki-build-image/Dockerfile index a163e9044080..9d52f1f1b602 100644 --- a/loki-build-image/Dockerfile +++ b/loki-build-image/Dockerfile @@ -15,6 +15,7 @@ RUN go get \ github.com/golang/protobuf/protoc-gen-go \ github.com/gogo/protobuf/protoc-gen-gogoslick \ github.com/gogo/protobuf/gogoproto \ + github.com/go-delve/delve/cmd/dlv \ golang.org/x/tools/cmd/goyacc && \ rm -rf /go/pkg /go/src ENV GOMETALINTER_VER="2.0.11"