diff --git a/.circleci/config.yml b/.circleci/config.yml index 6a0d1dc4f318..9f3c8e86c512 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,7 +41,7 @@ workflows: # https://circleci.com/blog/circleci-hacks-reuse-yaml-in-your-circleci-config-with-yaml/ defaults: &defaults docker: - - image: grafana/loki-build-image:0.1.0 + - image: grafana/loki-build-image:0.2.1 working_directory: /go/src/github.com/grafana/loki jobs: diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000000..e8e08c0eb805 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,19 @@ + + +**What this PR does / why we need it**: + +**Which issue(s) this PR fixes**: +Fixes # + +**Special notes for your reviewer**: + +**Checklist** +- [ ] Documentation added +- [ ] Tests updated + diff --git a/.gitignore b/.gitignore index dec47fd3a960..2c6409bbd5cd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .uptodate +.uptodate-debug .pkg .cache *.output @@ -7,6 +8,9 @@ requirements.lock mixin/vendor/ cmd/loki/loki cmd/promtail/promtail +cmd/loki/loki-debug +cmd/promtail/promtail-debug /loki /promtail /logcli +dlv \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 000000000000..2583383fdbcd --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,66 @@ +# This file contains all available configuration options +# with their default values. + +# options for analysis running +run: + # default concurrency is a available CPU number + concurrency: 2 + + # timeout for analysis, e.g. 30s, 5m, default is 1m + deadline: 5m + + # exit code when at least one issue was found, default is 1 + issues-exit-code: 1 + + # include test files or not, default is true + tests: true + + # list of build tags, all linters use it. Default is empty list. + build-tags: + + # which dirs to skip: they won't be analyzed; + # can use regexp here: generated.*, regexp is applied on full path; + # default value is empty list, but next dirs are always skipped independently + # from this option's value: + # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ + skip-dirs: + # which files to skip: they will be analyzed, but issues from them + # won't be reported. Default value is empty list, but there is + # no need to include all autogenerated files, we confidently recognize + # autogenerated files. If it's not please let us know. + skip-files: +# output configuration options +output: + # colored-line-number|line-number|json|tab|checkstyle, default is "colored-line-number" + format: colored-line-number + + # print lines of code with issue, default is true + print-issued-lines: true + + # print linter name in the end of issue text, default is true + print-linter-name: true + +linters: + enable: + - deadcode + - errcheck + - goconst + - gofmt + - goimports + - golint + - gosimple + - ineffassign + - megacheck + - misspell + - structcheck + - unconvert + - unparam + - varcheck + - govet + - unused # new from here. + - interfacer + - typecheck + +issues: + exclude: + - Error return value of .*log\.Logger\)\.Log\x60 is not checked \ No newline at end of file diff --git a/.gometalinter.json b/.gometalinter.json deleted file mode 100644 index ed3fda665df4..000000000000 --- a/.gometalinter.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "Vendor": true, - "Deadline": "5m", - "Concurrency": 2, - "Linters": { - "gofmt": {"Command": "gofmt -l -s -w"}, - "goimports": {"Command": "goimports -l -w"} - }, - "Exclude": [ - "\\.pb\\.go", - "method Seek.*should have signature", - "error return value not checked \\(level\\.", - "\"err\" shadows declaration" - ], - - "Enable": [ - "deadcode", - "errcheck", - "goconst", - "gofmt", - "goimports", - "golint", - "gosimple", - "gotypex", - "ineffassign", - "megacheck", - "misspell", - "structcheck", - "unconvert", - "unparam", - "varcheck", - "vet", - "vetshadow" - ] -} diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 000000000000..c6bffe79b060 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at contact@grafana.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 929b3bc7c5dc..9728341d22b1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,3 +18,8 @@ $ git remote add ``` Notice: `go get` return `package github.com/grafana/loki: no Go files in /go/src/github.com/grafana/loki` is normal. + + +## Contribute to helm + +Please follow [doc](./production/helm/README.md). diff --git a/Gopkg.lock b/Gopkg.lock index bc716a09b7cf..c31275b068b2 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -187,7 +187,7 @@ [[projects]] branch = "lazy-load-chunks" - digest = "1:bf1fa66c54722bc8664f1465e427cd6fe7df52f2b6fd5ab996baf37601687b70" + digest = "1:a999c29b3a215dfc12d374a9aac09c94c1b72ef530f4e39d9ab3ae1468cfe8e8" name = "github.com/cortexproject/cortex" packages = [ "pkg/chunk", @@ -213,7 +213,7 @@ "pkg/util/validation", ] pruneopts = "UT" - revision = "95a3f308e95617732b76e337874e83ccf173cf14" + revision = "61b92520b0c1afdef6e42b7a27cca6c715e9f386" source = "https://github.com/grafana/cortex" [[projects]] @@ -753,7 +753,7 @@ [[projects]] branch = "master" - digest = "1:c270fde3390cf824f004853135a8cb6347d013eab2d31e129bbae66219284284" + digest = "1:21e3d654236698240f954c8637759ba64af9b942e38a808579b8352772aa4ea6" name = "github.com/prometheus/prometheus" packages = [ "discovery", @@ -772,6 +772,7 @@ "discovery/zookeeper", "pkg/gate", "pkg/labels", + "pkg/modtimevfs", "pkg/relabel", "pkg/textparse", "pkg/timestamp", @@ -780,6 +781,7 @@ "relabel", "storage", "storage/tsdb", + "template", "util/stats", "util/strutil", "util/testutil", @@ -821,6 +823,26 @@ revision = "f0a61d5e8ca1bcc7a607d6de3dfd51467791db88" version = "v2.1.0" +[[projects]] + branch = "master" + digest = "1:d716544630de6b0d07c24bd4690547b6db84a8f62f71939a2957aec1a7b824d2" + name = "github.com/shurcooL/httpfs" + packages = [ + "filter", + "union", + "vfsutil", + ] + pruneopts = "UT" + revision = "74dc9339e414ad069a8d04bba7e7aafd08043a25" + +[[projects]] + branch = "master" + digest = "1:98450c86949b8cdc4637b80c1c686ca955e503d3fbae9296d1f49c532895d281" + name = "github.com/shurcooL/vfsgen" + packages = ["."] + pruneopts = "UT" + revision = "6a9ea43bcacdf716a5c1b38efff722c07adf0069" + [[projects]] digest = "1:87c2e02fb01c27060ccc5ba7c5a407cc91147726f8f40b70cceeedbc52b1f3a8" name = "github.com/sirupsen/logrus" @@ -878,8 +900,8 @@ version = "v2.0.0" [[projects]] - branch = "master" - digest = "1:81403343bc9a102e3c924f87ad81e5a13bb7c36211714475c906853f4e887933" + branch = "server-listen-addr" + digest = "1:0184d699d4cbbbc3073fbbdd7cc0c8592d484c6f23914c89fb6d218e90de171a" name = "github.com/weaveworks/common" packages = [ "aws", @@ -896,7 +918,8 @@ "user", ] pruneopts = "UT" - revision = "81a1a4d158e60de72dbead600ec011fb90344f8c" + revision = "5bf824591a6567784789cf9b2169f74f162bf80d" + source = "https://github.com/tomwilkie/weaveworks-common" [[projects]] digest = "1:bb40f7ff970145324f2a2acafdff3a23ed3f05db49cb5eb519b3d6bee86a5887" @@ -1388,9 +1411,15 @@ "github.com/prometheus/prometheus/discovery/config", "github.com/prometheus/prometheus/discovery/targetgroup", "github.com/prometheus/prometheus/pkg/labels", + "github.com/prometheus/prometheus/pkg/modtimevfs", "github.com/prometheus/prometheus/pkg/relabel", "github.com/prometheus/prometheus/pkg/textparse", + "github.com/prometheus/prometheus/promql", "github.com/prometheus/prometheus/relabel", + "github.com/prometheus/prometheus/template", + "github.com/shurcooL/httpfs/filter", + "github.com/shurcooL/httpfs/union", + "github.com/shurcooL/vfsgen", "github.com/stretchr/testify/assert", "github.com/stretchr/testify/require", "github.com/weaveworks/common/httpgrpc", diff --git a/Gopkg.toml b/Gopkg.toml index a190ef298052..35b9221f209b 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -31,7 +31,8 @@ [[constraint]] name = "github.com/weaveworks/common" - branch = "master" + source = "https://github.com/tomwilkie/weaveworks-common" + branch = "server-listen-addr" [[constraint]] name = "gopkg.in/fsnotify.v1" diff --git a/Makefile b/Makefile index e7c27e0f4189..cb060f57b03a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all test clean images protos +.PHONY: all test clean images protos assets check_assets .DEFAULT_GOAL := all CHARTS := production/helm/loki production/helm/promtail production/helm/loki-stack @@ -8,6 +8,7 @@ CHARTS := production/helm/loki production/helm/promtail production/helm/loki-sta IMAGE_PREFIX ?= grafana/ IMAGE_TAG := $(shell ./tools/image-tag) UPTODATE := .uptodate +DEBUG_UPTODATE := .uptodate-debug GIT_REVISION := $(shell git rev-parse --short HEAD) GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) @@ -21,6 +22,11 @@ GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) $(SUDO) docker tag $(IMAGE_PREFIX)$(shell basename $(@D)) $(IMAGE_PREFIX)$(shell basename $(@D)):$(IMAGE_TAG) touch $@ +%/$(DEBUG_UPTODATE): %/Dockerfile.debug + $(SUDO) docker build -f $(@D)/Dockerfile.debug -t $(IMAGE_PREFIX)$(shell basename $(@D))-debug $(@D)/ + $(SUDO) docker tag $(IMAGE_PREFIX)$(shell basename $(@D))-debug $(IMAGE_PREFIX)$(shell basename $(@D))-debug:$(IMAGE_TAG) + touch $@ + # We don't want find to scan inside a bunch of directories, to accelerate the # 'make: Entering directory '/go/src/github.com/grafana/loki' phase. DONT_FIND := -name tools -prune -o -name vendor -prune -o -name .git -prune -o -name .cache -prune -o -name .pkg -prune -o @@ -28,8 +34,13 @@ DONT_FIND := -name tools -prune -o -name vendor -prune -o -name .git -prune -o - # Get a list of directories containing Dockerfiles DOCKERFILES := $(shell find . $(DONT_FIND) -type f -name 'Dockerfile' -print) UPTODATE_FILES := $(patsubst %/Dockerfile,%/$(UPTODATE),$(DOCKERFILES)) +DEBUG_DOCKERFILES := $(shell find . $(DONT_FIND) -type f -name 'Dockerfile.debug' -print) +DEBUG_UPTODATE_FILES := $(patsubst %/Dockerfile.debug,%/$(DEBUG_UPTODATE),$(DEBUG_DOCKERFILES)) +DEBUG_DLV_FILES := $(patsubst %/Dockerfile.debug,%/dlv,$(DEBUG_DOCKERFILES)) DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile,%,$(DOCKERFILES)) IMAGE_NAMES := $(foreach dir,$(DOCKER_IMAGE_DIRS),$(patsubst %,$(IMAGE_PREFIX)%,$(shell basename $(dir)))) +DEBUG_DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile.debug,%,$(DEBUG_DOCKERFILES)) +DEBUG_IMAGE_NAMES := $(foreach dir,$(DEBUG_DOCKER_IMAGE_DIRS),$(patsubst %,$(IMAGE_PREFIX)%,$(shell basename $(dir))-debug)) images: $(info $(patsubst %,%:$(IMAGE_TAG),$(IMAGE_NAMES))) @echo > /dev/null @@ -49,22 +60,39 @@ YACC_GOS := $(patsubst %.y,%.go,$(YACC_DEFS)) # for every directory with main.go in it, in the ./cmd directory. MAIN_GO := $(shell find . $(DONT_FIND) -type f -name 'main.go' -print) EXES := $(foreach exe, $(patsubst ./cmd/%/main.go, %, $(MAIN_GO)), ./cmd/$(exe)/$(exe)) +DEBUG_EXES := $(foreach exe, $(patsubst ./cmd/%/main.go, %, $(MAIN_GO)), ./cmd/$(exe)/$(exe)-debug) GO_FILES := $(shell find . $(DONT_FIND) -name cmd -prune -o -type f -name '*.go' -print) + +# This is the important part of how `make all` enters this file +# the above EXES finds all the main.go files and for each of them +# it creates the dep_exe targets which look like this: +# cmd/promtail/promtail: loki-build-image/.uptodate cmd/promtail//main.go pkg/loki/loki.go pkg/loki/fake_auth.go ... +# cmd/promtail/.uptodate: cmd/promtail/promtail +# Then when `make all` expands `$(UPTODATE_FILES)` it will call the second generated target and initiate the build process define dep_exe $(1): $(dir $(1))/main.go $(GO_FILES) $(PROTO_GOS) $(YACC_GOS) $(dir $(1))$(UPTODATE): $(1) endef $(foreach exe, $(EXES), $(eval $(call dep_exe, $(exe)))) +# Everything is basically duplicated for debug builds, +# but with a different set of Dockerfiles and binaries appended with -debug. +define debug_dep_exe +$(1): $(dir $(1))/main.go $(GO_FILES) $(PROTO_GOS) $(YACC_GOS) +$(dir $(1))$(DEBUG_UPTODATE): $(1) +endef +$(foreach exe, $(DEBUG_EXES), $(eval $(call debug_dep_exe, $(exe)))) + # Manually declared dependancies and what goes into each exe pkg/logproto/logproto.pb.go: pkg/logproto/logproto.proto vendor/github.com/cortexproject/cortex/pkg/ring/ring.pb.go: vendor/github.com/cortexproject/cortex/pkg/ring/ring.proto vendor/github.com/cortexproject/cortex/pkg/ingester/client/cortex.pb.go: vendor/github.com/cortexproject/cortex/pkg/ingester/client/cortex.proto vendor/github.com/cortexproject/cortex/pkg/chunk/storage/caching_index_client.pb.go: vendor/github.com/cortexproject/cortex/pkg/chunk/storage/caching_index_client.proto -pkg/parser/labels.go: pkg/parser/labels.y -pkg/parser/matchers.go: pkg/parser/matchers.y +pkg/promtail/server/server.go: assets +pkg/logql/expr.go: pkg/logql/expr.y all: $(UPTODATE_FILES) test: $(PROTO_GOS) $(YACC_GOS) +debug: $(DEBUG_UPTODATE_FILES) yacc: $(YACC_GOS) protos: $(PROTO_GOS) yacc: $(YACC_GOS) @@ -86,6 +114,10 @@ TTY := --tty VPREFIX := github.com/grafana/loki/vendor/github.com/prometheus/common/version GO_FLAGS := -ldflags "-extldflags \"-static\" -s -w -X $(VPREFIX).Branch=$(GIT_BRANCH) -X $(VPREFIX).Version=$(IMAGE_TAG) -X $(VPREFIX).Revision=$(GIT_REVISION)" -tags netgo +# Per some websites I've seen to add `-gcflags "all=-N -l"`, the gcflags seem poorly if at all documented +# the best I could dig up is -N disables optimizations and -l disables inlining which should make debugging match source better. +# Also remove the -s and -w flags present in the normal build which strip the symbol table and the DWARF symbol table. +DEBUG_GO_FLAGS := -gcflags "all=-N -l" -ldflags "-extldflags \"-static\" -X $(VPREFIX).Branch=$(GIT_BRANCH) -X $(VPREFIX).Version=$(IMAGE_TAG) -X $(VPREFIX).Revision=$(GIT_REVISION)" -tags netgo NETGO_CHECK = @strings $@ | grep cgo_stub\\\.go >/dev/null || { \ rm $@; \ @@ -96,9 +128,15 @@ NETGO_CHECK = @strings $@ | grep cgo_stub\\\.go >/dev/null || { \ false; \ } +# If BUILD_IN_CONTAINER is true, the build image is run which launches +# an image that mounts this project as a volume. The image invokes a build.sh script +# which essentially re-enters this file with BUILD_IN_CONTAINER=FALSE +# causing the else block target below to be called and the files to be built. +# If BUILD_IN_CONTAINER were false to begin with, the else block is +# executed and the binaries are built without ever launching the build container. ifeq ($(BUILD_IN_CONTAINER),true) -$(EXES) $(PROTO_GOS) $(YACC_GOS) lint test shell check-generated-files: loki-build-image/$(UPTODATE) +$(EXES) $(DEBUG_EXES) $(PROTO_GOS) $(YACC_GOS) lint test shell check-generated-files: loki-build-image/$(UPTODATE) @mkdir -p $(shell pwd)/.pkg @mkdir -p $(shell pwd)/.cache $(SUDO) docker run $(RM) $(TTY) -i \ @@ -109,6 +147,12 @@ $(EXES) $(PROTO_GOS) $(YACC_GOS) lint test shell check-generated-files: loki-bui else +$(DEBUG_EXES): loki-build-image/$(UPTODATE) + CGO_ENABLED=0 go build $(DEBUG_GO_FLAGS) -o $@ ./$(@D) + $(NETGO_CHECK) + # Copy the delve binary to make it easily available to put in the binary's container. + [ -f "/go/bin/dlv" ] && mv "/go/bin/dlv" $(@D)/dlv + $(EXES): loki-build-image/$(UPTODATE) CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D) $(NETGO_CHECK) @@ -127,13 +171,13 @@ $(EXES): loki-build-image/$(UPTODATE) goyacc -p $(basename $(notdir $<)) -o $@ $< lint: loki-build-image/$(UPTODATE) - gometalinter ./... + GOGC=20 golangci-lint run check-generated-files: loki-build-image/$(UPTODATE) yacc protos @git diff-files || (echo "changed files; failing check" && exit 1) test: loki-build-image/$(UPTODATE) - go test ./... + go test -p=8 ./... shell: loki-build-image/$(UPTODATE) bash @@ -200,6 +244,17 @@ helm-publish: helm git push origin gh-pages clean: - $(SUDO) docker rmi $(IMAGE_NAMES) >/dev/null 2>&1 || true - rm -rf $(UPTODATE_FILES) $(EXES) .cache + $(SUDO) docker rmi $(IMAGE_NAMES) $(DEBUG_IMAGE_NAMES) >/dev/null 2>&1 || true + rm -rf $(UPTODATE_FILES) $(EXES) $(DEBUG_UPTODATE_FILES) $(DEBUG_EXES) $(DEBUG_DLV_FILES) .cache pkg/promtail/server/ui/assets_vfsdata.go go clean ./... + +assets: + @echo ">> writing assets" + go generate -x -v ./pkg/promtail/server/ui + +check_assets: assets + @echo ">> checking that assets are up-to-date" + @if ! (cd pkg/promtail/server/ui && git diff --exit-code); then \ + echo "Run 'make assets' and commit the changes to fix the error."; \ + exit 1; \ + fi diff --git a/README.md b/README.md index fc4c72a9d242..5703f7291a55 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Once you have promtail, Loki, and Grafana running, continue with [our usage docs - [API documentation](./docs/api.md) for alternative ways of getting logs into Loki. - [Operations](./docs/operations.md) for important aspects of running Loki. -- [Promtail](./docs/promtail-setup.md) on how to configure the agent that tails your logs. +- [Promtail](./docs/promtail.md) is an agent which can tail your log files and push them to Loki. - [Logcli](./docs/logcli.md) on how to query your logs without Grafana. - [Troubleshooting](./docs/troubleshooting.md) for help around frequent error messages. - [Usage](./docs/usage.md) for how to set up a Loki datasource in Grafana and query your logs. @@ -55,12 +55,16 @@ Your feedback is always welcome. ## Further Reading - The original [design doc](https://docs.google.com/document/d/11tjK_lvp1-SVsFZjgOTr1vV3-q6vBAsZYIQ5ZeYBkyM/view) for Loki is a good source for discussion of the motivation and design decisions. +- Callum Styan's March 2019 DevOpsDays Vancouver talk "[Grafana Loki: Log Aggregation for Incident Investigations][devopsdays19-talk]". +- Grafana Labs blog post "[How We Designed Loki to Work Easily Both as Microservices and as Monoliths][architecture-blog]". - Julien Garcia Gonzalez' March 2019 blog post "[Grafana Logging using Loki][giant-swarm-blog]". - Tom Wilkie's early-2019 CNCF Paris/FODEM talk "[Grafana Loki: like Prometheus, but for logs][fosdem19-talk]" ([slides][fosdem19-slides], [video][fosdem19-video]). - David Kaltschmidt's KubeCon 2018 talk "[On the OSS Path to Full Observability with Grafana][kccna18-event]" ([slides][kccna18-slides], [video][kccna18-video]) on how Loki fits into a cloud-native environment. - Goutham Veeramachaneni's blog post "[Loki: Prometheus-inspired, open source logging for cloud natives](https://grafana.com/blog/2018/12/12/loki-prometheus-inspired-open-source-logging-for-cloud-natives/)" on details of the Loki architectire. - David Kaltschmidt's blog post "[Closer look at Grafana's user interface for Loki](https://grafana.com/blog/2019/01/02/closer-look-at-grafanas-user-interface-for-loki/)" on the ideas that went into the logging user interface. +[devopsdays19-talk]: https://grafana.com/blog/2019/05/06/how-loki-correlates-metrics-and-logs----and-saves-you-money/ +[architecture-blog]: https://grafana.com/blog/2019/04/15/how-we-designed-loki-to-work-easily-both-as-microservices-and-as-monoliths/ [giant-swarm-blog]: https://blog.giantswarm.io/grafana-logging-using-loki [fosdem19-talk]: https://fosdem.org/2019/schedule/event/loki_prometheus_for_logs/ [fosdem19-slides]: https://speakerdeck.com/grafana/grafana-loki-like-prometheus-but-for-logs diff --git a/cmd/logcli/client.go b/cmd/logcli/client.go index 3acaea6b65ee..59b5754f914b 100644 --- a/cmd/logcli/client.go +++ b/cmd/logcli/client.go @@ -12,6 +12,7 @@ import ( "time" "github.com/gorilla/websocket" + "github.com/prometheus/common/config" "github.com/grafana/loki/pkg/logproto" ) @@ -60,15 +61,31 @@ func doRequest(path string, out interface{}) error { if err != nil { return err } + req.SetBasicAuth(*username, *password) - resp, err := http.DefaultClient.Do(req) + clientConfig := config.HTTPClientConfig{ + TLSConfig: config.TLSConfig{ + CAFile: *tlsCACertPath, + CertFile: *tlsClientCertPath, + KeyFile: *tlsClientCertKeyPath, + ServerName: url, + InsecureSkipVerify: *tlsSkipVerify, + }, + } + + client, err := config.NewClientFromConfig(clientConfig, "logcli") + if err != nil { + return err + } + + resp, err := client.Do(req) if err != nil { return err } defer func() { if err := resp.Body.Close(); err != nil { - fmt.Println("error closing body", err) + log.Println("error closing body", err) } }() @@ -86,16 +103,33 @@ func liveTailQueryConn() (*websocket.Conn, error) { } func wsConnect(path string) (*websocket.Conn, error) { + + tlsConfig, err := config.NewTLSConfig(&config.TLSConfig{ + CAFile: *tlsCACertPath, + CertFile: *tlsClientCertPath, + KeyFile: *tlsClientCertKeyPath, + ServerName: *addr, + InsecureSkipVerify: *tlsSkipVerify, + }) + if err != nil { + return nil, err + } + url := *addr + path if strings.HasPrefix(url, "https") { url = strings.Replace(url, "https", "wss", 1) } else if strings.HasPrefix(url, "http") { url = strings.Replace(url, "http", "ws", 1) } - fmt.Println(url) + log.Println(url) h := http.Header{"Authorization": {"Basic " + base64.StdEncoding.EncodeToString([]byte(*username+":"+*password))}} - c, resp, err := websocket.DefaultDialer.Dial(url, h) + + ws := websocket.Dialer{ + TLSClientConfig: tlsConfig, + } + + c, resp, err := ws.Dial(url, h) if err != nil { if resp == nil { diff --git a/cmd/logcli/main.go b/cmd/logcli/main.go index 596bdda6f0f0..d37074bc18ab 100644 --- a/cmd/logcli/main.go +++ b/cmd/logcli/main.go @@ -14,14 +14,22 @@ var ( username = app.Flag("username", "Username for HTTP basic auth.").Default("").Envar("GRAFANA_USERNAME").String() password = app.Flag("password", "Password for HTTP basic auth.").Default("").Envar("GRAFANA_PASSWORD").String() - queryCmd = app.Command("query", "Run a LogQL query.") - queryStr = queryCmd.Arg("query", "eg '{foo=\"bar\",baz=\"blip\"}'").Required().String() - regexpStr = queryCmd.Arg("regex", "").String() - limit = queryCmd.Flag("limit", "Limit on number of entries to print.").Default("30").Int() - since = queryCmd.Flag("since", "Lookback window.").Default("1h").Duration() - forward = queryCmd.Flag("forward", "Scan forwards through logs.").Default("false").Bool() - tail = queryCmd.Flag("tail", "Tail the logs").Short('t').Default("false").Bool() - noLabels = queryCmd.Flag("no-labels", "Do not print labels").Default("false").Bool() + tlsCACertPath = app.Flag("ca-cert", "Path to the server Certificate Authority.").Default("").Envar("LOKI_CA_CERT_PATH").String() + tlsSkipVerify = app.Flag("tls-skip-verify", "Server certificate TLS skip verify.").Default("false").Bool() + tlsClientCertPath = app.Flag("cert", "Path to the client certificate.").Default("").Envar("LOKI_CLIENT_CERT_PATH").String() + tlsClientCertKeyPath = app.Flag("key", "Path to the client certificate key.").Default("").Envar("LOKI_CLIENT_KEY_PATH").String() + + queryCmd = app.Command("query", "Run a LogQL query.") + queryStr = queryCmd.Arg("query", "eg '{foo=\"bar\",baz=\"blip\"}'").Required().String() + regexpStr = queryCmd.Arg("regex", "").String() + limit = queryCmd.Flag("limit", "Limit on number of entries to print.").Default("30").Int() + since = queryCmd.Flag("since", "Lookback window.").Default("1h").Duration() + forward = queryCmd.Flag("forward", "Scan forwards through logs.").Default("false").Bool() + tail = queryCmd.Flag("tail", "Tail the logs").Short('t').Default("false").Bool() + noLabels = queryCmd.Flag("no-labels", "Do not print any labels").Default("false").Bool() + ignoreLabelsKey = queryCmd.Flag("exclude-label", "Exclude labels given the provided key during output.").Strings() + showLabelsKey = queryCmd.Flag("include-label", "Include labels given the provided key during output.").Strings() + fixedLabelsLen = queryCmd.Flag("labels-length", "Set a fixed padding to labels").Default("0").Int() labelsCmd = app.Command("labels", "Find values for a given label.") labelName = labelsCmd.Arg("label", "The name of the label.").HintAction(listLabels).String() diff --git a/cmd/logcli/query.go b/cmd/logcli/query.go index 4a7b47f0d034..f5b571627947 100644 --- a/cmd/logcli/query.go +++ b/cmd/logcli/query.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "log" "strings" "time" @@ -11,7 +10,6 @@ import ( "github.com/grafana/loki/pkg/iter" "github.com/grafana/loki/pkg/logproto" - "github.com/grafana/loki/pkg/parser" ) func doQuery() { @@ -21,9 +19,8 @@ func doQuery() { } var ( - i iter.EntryIterator - common labels.Labels - maxLabelsLen = 100 + i iter.EntryIterator + common labels.Labels ) end := time.Now() @@ -43,24 +40,43 @@ func doQuery() { labelsCache := func(labels string) labels.Labels { return cache[labels] } + common = commonLabels(lss) - i = iter.NewQueryResponseIterator(resp, d) + + // Remove the labels we want to show from common + if len(*showLabelsKey) > 0 { + common = common.MatchLabels(false, *showLabelsKey...) + } if len(common) > 0 { - fmt.Println("Common labels:", color.RedString(common.String())) + log.Println("Common labels:", color.RedString(common.String())) + } + + if len(*ignoreLabelsKey) > 0 { + log.Println("Ignoring labels key:", color.RedString(strings.Join(*ignoreLabelsKey, ","))) } + // Get the max size of labels + maxLabelsLen := *fixedLabelsLen for _, ls := range cache { ls = subtract(common, ls) + if len(*ignoreLabelsKey) > 0 { + ls = ls.MatchLabels(false, *ignoreLabelsKey...) + } len := len(ls.String()) if maxLabelsLen < len { maxLabelsLen = len } } + i = iter.NewQueryResponseIterator(resp, d) + for i.Next() { ls := labelsCache(i.Labels()) ls = subtract(ls, common) + if len(*ignoreLabelsKey) > 0 { + ls = ls.MatchLabels(false, *ignoreLabelsKey...) + } labels := "" if !*noLabels { @@ -74,95 +90,3 @@ func doQuery() { log.Fatalf("Error from iterator: %v", err) } } - -func printLogEntry(ts time.Time, lbls string, line string) { - fmt.Println( - color.BlueString(ts.Format(time.RFC3339)), - color.RedString(lbls), - strings.TrimSpace(line), - ) -} - -func padLabel(ls labels.Labels, maxLabelsLen int) string { - labels := ls.String() - if len(labels) < maxLabelsLen { - labels += strings.Repeat(" ", maxLabelsLen-len(labels)) - } - return labels -} - -func mustParseLabels(labels string) labels.Labels { - ls, err := parser.Labels(labels) - if err != nil { - log.Fatalf("Failed to parse labels: %+v", err) - } - return ls -} - -func parseLabels(resp *logproto.QueryResponse) (map[string]labels.Labels, []labels.Labels) { - cache := make(map[string]labels.Labels, len(resp.Streams)) - lss := make([]labels.Labels, 0, len(resp.Streams)) - for _, stream := range resp.Streams { - ls := mustParseLabels(stream.Labels) - cache[stream.Labels] = ls - lss = append(lss, ls) - } - return cache, lss -} - -func commonLabels(lss []labels.Labels) labels.Labels { - if len(lss) == 0 { - return nil - } - - result := lss[0] - for i := 1; i < len(lss); i++ { - result = intersect(result, lss[i]) - } - return result -} - -func intersect(a, b labels.Labels) labels.Labels { - var result labels.Labels - for i, j := 0, 0; i < len(a) && j < len(b); { - k := strings.Compare(a[i].Name, b[j].Name) - switch { - case k == 0: - if a[i].Value == b[j].Value { - result = append(result, a[i]) - } - i++ - j++ - case k < 0: - i++ - case k > 0: - j++ - } - } - return result -} - -// subtract b from a -func subtract(a, b labels.Labels) labels.Labels { - var result labels.Labels - i, j := 0, 0 - for i < len(a) && j < len(b) { - k := strings.Compare(a[i].Name, b[j].Name) - if k != 0 || a[i].Value != b[j].Value { - result = append(result, a[i]) - } - switch { - case k == 0: - i++ - j++ - case k < 0: - i++ - case k > 0: - j++ - } - } - for ; i < len(a); i++ { - result = append(result, a[i]) - } - return result -} diff --git a/cmd/logcli/query_test.go b/cmd/logcli/query_test.go new file mode 100644 index 000000000000..a8a3db231df9 --- /dev/null +++ b/cmd/logcli/query_test.go @@ -0,0 +1,130 @@ +package main + +import ( + "reflect" + "testing" + + "github.com/prometheus/prometheus/pkg/labels" +) + +func Test_commonLabels(t *testing.T) { + type args struct { + lss []labels.Labels + } + tests := []struct { + name string + args args + want labels.Labels + }{ + { + "Extract common labels source > target", + args{ + []labels.Labels{mustParseLabels(`{foo="bar", bar="foo"}`), mustParseLabels(`{bar="foo", foo="foo", baz="baz"}`)}, + }, + mustParseLabels(`{bar="foo"}`), + }, + { + "Extract common labels source > target", + args{ + []labels.Labels{mustParseLabels(`{foo="bar", bar="foo"}`), mustParseLabels(`{bar="foo", foo="bar", baz="baz"}`)}, + }, + mustParseLabels(`{foo="bar", bar="foo"}`), + }, + { + "Extract common labels source < target", + args{ + []labels.Labels{mustParseLabels(`{foo="bar", bar="foo"}`), mustParseLabels(`{bar="foo"}`)}, + }, + mustParseLabels(`{bar="foo"}`), + }, + { + "Extract common labels source < target no common", + args{ + []labels.Labels{mustParseLabels(`{foo="bar", bar="foo"}`), mustParseLabels(`{fo="bar"}`)}, + }, + labels.Labels{}, + }, + { + "Extract common labels source = target no common", + args{ + []labels.Labels{mustParseLabels(`{foo="bar"}`), mustParseLabels(`{fooo="bar"}`)}, + }, + labels.Labels{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := commonLabels(tt.args.lss); !reflect.DeepEqual(got, tt.want) { + t.Errorf("commonLabels() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_subtract(t *testing.T) { + type args struct { + a labels.Labels + b labels.Labels + } + tests := []struct { + name string + args args + want labels.Labels + }{ + { + "Subtract labels source > target", + args{ + mustParseLabels(`{foo="bar", bar="foo"}`), + mustParseLabels(`{bar="foo", foo="foo", baz="baz"}`), + }, + mustParseLabels(`{foo="bar"}`), + }, + { + "Subtract labels source < target", + args{ + mustParseLabels(`{foo="bar", bar="foo"}`), + mustParseLabels(`{bar="foo"}`), + }, + mustParseLabels(`{foo="bar"}`), + }, + { + "Subtract labels source < target no sub", + args{ + mustParseLabels(`{foo="bar", bar="foo"}`), + mustParseLabels(`{fo="bar"}`), + }, + mustParseLabels(`{bar="foo", foo="bar"}`), + }, + { + "Subtract labels source = target no sub", + args{ + mustParseLabels(`{foo="bar"}`), + mustParseLabels(`{fiz="buz"}`), + }, + mustParseLabels(`{foo="bar"}`), + }, + { + "Subtract labels source > target no sub", + args{ + mustParseLabels(`{foo="bar"}`), + mustParseLabels(`{fiz="buz", foo="baz"}`), + }, + mustParseLabels(`{foo="bar"}`), + }, + { + "Subtract labels source > target no sub", + args{ + mustParseLabels(`{a="b", foo="bar", baz="baz", fizz="fizz"}`), + mustParseLabels(`{foo="bar", baz="baz", buzz="buzz", fizz="fizz"}`), + }, + mustParseLabels(`{a="b"}`), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := subtract(tt.args.a, tt.args.b); !reflect.DeepEqual(got, tt.want) { + t.Errorf("subtract() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/logcli/tail.go b/cmd/logcli/tail.go index 51e64bd221e0..3f9a6806251a 100644 --- a/cmd/logcli/tail.go +++ b/cmd/logcli/tail.go @@ -2,8 +2,11 @@ package main import ( "log" + "strings" - "github.com/grafana/loki/pkg/logproto" + "github.com/grafana/loki/pkg/querier" + + "github.com/fatih/color" ) func tailQuery() { @@ -12,21 +15,49 @@ func tailQuery() { log.Fatalf("Tailing logs failed: %+v", err) } - stream := new(logproto.Stream) + tailReponse := new(querier.TailResponse) + + if len(*ignoreLabelsKey) > 0 { + log.Println("Ingoring labels key:", color.RedString(strings.Join(*ignoreLabelsKey, ","))) + } + + if len(*showLabelsKey) > 0 { + log.Println("Print only labels key:", color.RedString(strings.Join(*showLabelsKey, ","))) + } for { - err := conn.ReadJSON(stream) + err := conn.ReadJSON(tailReponse) if err != nil { log.Println("Error reading stream:", err) return } labels := "" - if !*noLabels { - labels = stream.Labels - } - for _, entry := range stream.Entries { - printLogEntry(entry.Timestamp, labels, entry.Line) + for _, stream := range tailReponse.Streams { + if !*noLabels { + + if len(*ignoreLabelsKey) > 0 || len(*showLabelsKey) > 0 { + + ls := mustParseLabels(stream.GetLabels()) + + if len(*showLabelsKey) > 0 { + ls = ls.MatchLabels(true, *showLabelsKey...) + } + + if len(*ignoreLabelsKey) > 0 { + ls = ls.MatchLabels(false, *ignoreLabelsKey...) + } + + labels = ls.String() + + } else { + + labels = stream.Labels + } + } + for _, entry := range stream.Entries { + printLogEntry(entry.Timestamp, labels, entry.Line) + } } } } diff --git a/cmd/logcli/utils.go b/cmd/logcli/utils.go new file mode 100644 index 000000000000..807c3a618487 --- /dev/null +++ b/cmd/logcli/utils.go @@ -0,0 +1,109 @@ +package main + +import ( + "fmt" + "log" + "sort" + "strings" + "time" + + "github.com/fatih/color" + "github.com/grafana/loki/pkg/logproto" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/promql" +) + +// print a log entry +func printLogEntry(ts time.Time, lbls string, line string) { + fmt.Println( + color.BlueString(ts.Format(time.RFC3339)), + color.RedString(lbls), + strings.TrimSpace(line), + ) +} + +// add some padding after labels +func padLabel(ls labels.Labels, maxLabelsLen int) string { + labels := ls.String() + if len(labels) < maxLabelsLen { + labels += strings.Repeat(" ", maxLabelsLen-len(labels)) + } + return labels +} + +// parse labels from string +func mustParseLabels(labels string) labels.Labels { + ls, err := promql.ParseMetric(labels) + if err != nil { + log.Fatalf("Failed to parse labels: %+v", err) + } + return ls +} + +// parse labels from response stream +func parseLabels(resp *logproto.QueryResponse) (map[string]labels.Labels, []labels.Labels) { + cache := make(map[string]labels.Labels, len(resp.Streams)) + lss := make([]labels.Labels, 0, len(resp.Streams)) + for _, stream := range resp.Streams { + ls := mustParseLabels(stream.Labels) + cache[stream.Labels] = ls + lss = append(lss, ls) + } + return cache, lss +} + +// return common labels between given lavels set +func commonLabels(lss []labels.Labels) labels.Labels { + if len(lss) == 0 { + return nil + } + + result := lss[0] + for i := 1; i < len(lss); i++ { + result = intersect(result, lss[i]) + } + return result +} + +// intersect two labels set +func intersect(a, b labels.Labels) labels.Labels { + + set := labels.Labels{} + ma := a.Map() + mb := b.Map() + + for ka, va := range ma { + if vb, ok := mb[ka]; ok { + if vb == va { + set = append(set, labels.Label{ + Name: ka, + Value: va, + }) + } + } + } + sort.Sort(set) + return set +} + +// subtract labels set b from labels set a +func subtract(a, b labels.Labels) labels.Labels { + + set := labels.Labels{} + ma := a.Map() + mb := b.Map() + + for ka, va := range ma { + if vb, ok := mb[ka]; ok { + if vb == va { + continue + } + } + set = append(set, labels.Label{ + Name: ka, + Value: va, + }) + } + sort.Sort(set) + return set +} diff --git a/cmd/loki/Dockerfile.debug b/cmd/loki/Dockerfile.debug new file mode 100644 index 000000000000..ca4c61f86eec --- /dev/null +++ b/cmd/loki/Dockerfile.debug @@ -0,0 +1,17 @@ +FROM alpine:3.9 +RUN apk add --update --no-cache ca-certificates +COPY loki-debug /bin/loki-debug +ADD dlv /usr/bin +COPY loki-local-config.yaml /etc/loki/local-config.yaml +EXPOSE 80 + +# Expose 40000 for delve +EXPOSE 40000 + +# Allow delve to run on Alpine based containers. +RUN apk add --no-cache libc6-compat + +# Run delve, ending with -- because we pass params via kubernetes, per the docs: +# Pass flags to the program you are debugging using --, for example:` +# dlv exec ./hello -- server --config conf/config.toml` +ENTRYPOINT ["/usr/bin/dlv", "--listen=:40000", "--headless=true", "--api-version=2", "exec", "/bin/loki-debug", "--"] diff --git a/cmd/loki/loki-local-config.yaml b/cmd/loki/loki-local-config.yaml index 24d7a8344a52..3962fb37edec 100644 --- a/cmd/loki/loki-local-config.yaml +++ b/cmd/loki/loki-local-config.yaml @@ -30,3 +30,6 @@ storage_config: limits_config: enforce_metric_name: false + +querier: + max_look_back_period: 0 diff --git a/cmd/promtail/Dockerfile.debug b/cmd/promtail/Dockerfile.debug new file mode 100644 index 000000000000..48a0a2113b5d --- /dev/null +++ b/cmd/promtail/Dockerfile.debug @@ -0,0 +1,17 @@ +FROM alpine:3.9 +RUN apk add --update --no-cache ca-certificates +ADD promtail-debug /usr/bin +ADD dlv /usr/bin +COPY promtail-local-config.yaml /etc/promtail/local-config.yaml +COPY promtail-docker-config.yaml /etc/promtail/docker-config.yaml + +# Expose 40000 for delve +EXPOSE 40000 + +# Allow delve to run on Alpine based containers. +RUN apk add --no-cache libc6-compat + +# Run delve, ending with -- because we pass params via kubernetes, per the docs: +# Pass flags to the program you are debugging using --, for example:` +# dlv exec ./hello -- server --config conf/config.toml` +ENTRYPOINT ["/usr/bin/dlv", "--listen=:40000", "--headless=true", "--api-version=2", "exec", "/usr/bin/promtail-debug", "--"] \ No newline at end of file diff --git a/cmd/promtail/main.go b/cmd/promtail/main.go index 77ece24615fe..ba9fb026d9e9 100644 --- a/cmd/promtail/main.go +++ b/cmd/promtail/main.go @@ -28,7 +28,7 @@ func main() { flagext.RegisterFlags(&config) flag.Parse() - util.InitLogger(&config.ServerConfig) + util.InitLogger(&config.ServerConfig.Config) if configFile != "" { if err := helpers.LoadConfig(configFile, &config); err != nil { @@ -37,6 +37,9 @@ func main() { } } + // Re-init the logger which will now honor a different log level set in ServerConfig.Config + util.InitLogger(&config.ServerConfig.Config) + p, err := promtail.New(config) if err != nil { level.Error(util.Logger).Log("msg", "error creating promtail", "error", err) diff --git a/cmd/promtail/promtail-docker-config.yaml b/cmd/promtail/promtail-docker-config.yaml index 8dc8f27307c7..b14c23009a29 100644 --- a/cmd/promtail/promtail-docker-config.yaml +++ b/cmd/promtail/promtail-docker-config.yaml @@ -5,8 +5,8 @@ server: positions: filename: /tmp/positions.yaml -client: - url: http://loki:3100/api/prom/push +clients: + - url: http://loki:3100/api/prom/push scrape_configs: - job_name: system diff --git a/cmd/promtail/promtail-local-config.yaml b/cmd/promtail/promtail-local-config.yaml index d5b13c5e0c23..d2f34c5fe599 100644 --- a/cmd/promtail/promtail-local-config.yaml +++ b/cmd/promtail/promtail-local-config.yaml @@ -5,8 +5,8 @@ server: positions: filename: /tmp/positions.yaml -client: - url: http://localhost:3100/api/prom/push +clients: + - url: http://localhost:3100/api/prom/push scrape_configs: - job_name: system diff --git a/debug/README.md b/debug/README.md new file mode 100644 index 000000000000..b4e571325a2a --- /dev/null +++ b/debug/README.md @@ -0,0 +1,57 @@ +## Debug images + +To build debug images run + +```shell +make debug +``` + +You can use the `docker-compose.yaml` in this directory to launch the debug versions of the image in docker + + +## Promtail in kubernetes + +If you want to debug promtail in kubernetes, I have done so with the ksonnet setup: + +```shell +ks init promtail +cd promtail +ks env add promtail +jb init +jb install github.com/grafana/loki/production/ksonnet/promtail +vi environments/promtail/main.jsonnet +``` + +Replace the contents with: + +```jsonnet +local promtail = import 'promtail/promtail.libsonnet'; + + +promtail + { + _images+:: { + promtail: 'grafana/promtail-debug:latest', + }, + _config+:: { + namespace: 'default', + + promtail_config+: { + external_labels+: { + cluster: 'some_cluster_name', + }, + scheme: 'https', + hostname: 'hostname', + username: 'username', + password: 'password', + }, + }, +} +``` + +change the `some_cluster_name` to anything meaningful to help find your logs in loki + +also update the `hostname`, `username`, and `password` for your loki instance. + +## Loki in kubernetes + +Haven't tried this yet, it works from docker-compose so it should run in kubernetes just fine also. \ No newline at end of file diff --git a/debug/docker-compose.yaml b/debug/docker-compose.yaml new file mode 100644 index 000000000000..88c0811ac125 --- /dev/null +++ b/debug/docker-compose.yaml @@ -0,0 +1,37 @@ +version: "3" + +networks: + loki: + +services: + loki: + # this is required according to https://github.com/Microsoft/vscode-go/wiki/Debugging-Go-code-using-VS-Code#linuxdocker + security_opt: + - seccomp:unconfined + image: grafana/loki-debug:latest + ports: + - "40000:40000" + - "3100:3100" + command: -config.file=/etc/loki/local-config.yaml + networks: + - loki + + promtail: + # this is required according to https://github.com/Microsoft/vscode-go/wiki/Debugging-Go-code-using-VS-Code#linuxdocker + security_opt: + - seccomp:unconfined + image: grafana/promtail-debug:latest + ports: + - "40100:40000" + volumes: + - /var/log:/var/log + command: -config.file=/etc/promtail/docker-config.yaml + networks: + - loki + + grafana: + image: grafana/grafana:master + ports: + - "3000:3000" + networks: + - loki diff --git a/docs/logcli.md b/docs/logcli.md index d7330a2826bb..678fb37b8817 100644 --- a/docs/logcli.md +++ b/docs/logcli.md @@ -44,8 +44,8 @@ Common labels: {job="cortex-ops/consul", namespace="cortex-ops"} ### Configuration - Configuration values are considered in the following order (lowest to highest): + - environment value - command line @@ -53,17 +53,22 @@ The URLs of the requests are printed to help with integration work. ### Details -``` +```console $ logcli help usage: logcli [] [ ...] A command-line for loki. Flags: - --help Show context-sensitive help (also try --help-long and --help-man). - --addr="" Server address, need to specify. - --username="" Username for HTTP basic auth. - --password="" Password for HTTP basic auth. + --help Show context-sensitive help (also try --help-long and --help-man). + --addr="https://logs-us-west1.grafana.net" + Server address. + --username="" Username for HTTP basic auth. + --password="" Password for HTTP basic auth. + --ca-cert="" Path to the server Certificate Authority. + --tls-skip-verify Server certificate TLS skip verify. + --cert="" Path to the client certificate. + --key="" Path to the client certificate key. Commands: help [...] @@ -72,7 +77,7 @@ Commands: query [] [] Run a LogQL query. - labels