diff --git a/cmd/logcli/main.go b/cmd/logcli/main.go index 3d2aa85297b3..e4e04da8d665 100644 --- a/cmd/logcli/main.go +++ b/cmd/logcli/main.go @@ -16,6 +16,7 @@ import ( "gopkg.in/alecthomas/kingpin.v2" "github.com/grafana/loki/v3/pkg/logcli/client" + "github.com/grafana/loki/v3/pkg/logcli/detected" "github.com/grafana/loki/v3/pkg/logcli/index" "github.com/grafana/loki/v3/pkg/logcli/labelquery" "github.com/grafana/loki/v3/pkg/logcli/output" @@ -253,6 +254,39 @@ Example: 'my-query' `) volumeRangeQuery = newVolumeQuery(true, volumeRangeCmd) + + detectedFieldsCmd = app.Command("detected-fields", `Run a query for detected fields.. + +The "detected-fields" command will return information about fields detected using either +the "logfmt" or "json" parser against the log lines returned by the provided query for the +provided time range. + +The "detected-fields" command will output extra information about the query +and its results, such as the API URL, set of common labels, and set +of excluded labels. This extra information can be suppressed with the +--quiet flag. + +By default we look over the last hour of data; use --since to modify +or provide specific start and end times with --from and --to respectively. + +Notice that when using --from and --to then ensure to use RFC3339Nano +time format, but without timezone at the end. The local timezone will be added +automatically or if using --timezone flag. + +Example: + + logcli detected-fields + --timezone=UTC + --from="2021-01-19T10:00:00Z" + --to="2021-01-19T20:00:00Z" + --output=jsonl + 'my-query' + +The output is limited to 100 fields by default; use --field-limit to increase. +The query is limited to processing 1000 lines per subquery; use --line-limit to increase. +`) + + detectedFieldsQuery = newDetectedFieldsQuery(detectedFieldsCmd) ) func main() { @@ -388,6 +422,8 @@ func main() { } else { index.GetVolume(volumeQuery, queryClient, out, *statistics) } + case detectedFieldsCmd.FullCommand(): + detectedFieldsQuery.Do(queryClient, *outputMode) } } @@ -652,3 +688,41 @@ func newVolumeQuery(rangeQuery bool, cmd *kingpin.CmdClause) *volume.Query { return q } + +func newDetectedFieldsQuery(cmd *kingpin.CmdClause) *detected.FieldsQuery { + // calculate query range from cli params + var from, to string + var since time.Duration + + q := &detected.FieldsQuery{} + + // executed after all command flags are parsed + cmd.Action(func(c *kingpin.ParseContext) error { + defaultEnd := time.Now() + defaultStart := defaultEnd.Add(-since) + + q.Start = mustParse(from, defaultStart) + q.End = mustParse(to, defaultEnd) + + q.Quiet = *quiet + + return nil + }) + + cmd.Flag("field-limit", "Limit on number of fields to return."). + Default("100"). + IntVar(&q.FieldLimit) + cmd.Flag("line-limit", "Limit the number of lines each subquery is allowed to process."). + Default("1000"). + IntVar(&q.LineLimit) + cmd.Arg("query", "eg '{foo=\"bar\",baz=~\".*blip\"} |~ \".*error.*\"'"). + Required(). + StringVar(&q.QueryString) + cmd.Flag("since", "Lookback window.").Default("1h").DurationVar(&since) + cmd.Flag("from", "Start looking for logs at this absolute time (inclusive)").StringVar(&from) + cmd.Flag("to", "Stop looking for logs at this absolute time (exclusive)").StringVar(&to) + cmd.Flag("step", "Query resolution step width, for metric queries. Evaluate the query at the specified step over the time range."). + DurationVar(&q.Step) + + return q +} diff --git a/pkg/logcli/client/client.go b/pkg/logcli/client/client.go index 73ddccd7efd1..e417ccfa3ce5 100644 --- a/pkg/logcli/client/client.go +++ b/pkg/logcli/client/client.go @@ -28,16 +28,17 @@ import ( ) const ( - queryPath = "/loki/api/v1/query" - queryRangePath = "/loki/api/v1/query_range" - labelsPath = "/loki/api/v1/labels" - labelValuesPath = "/loki/api/v1/label/%s/values" - seriesPath = "/loki/api/v1/series" - tailPath = "/loki/api/v1/tail" - statsPath = "/loki/api/v1/index/stats" - volumePath = "/loki/api/v1/index/volume" - volumeRangePath = "/loki/api/v1/index/volume_range" - defaultAuthHeader = "Authorization" + queryPath = "/loki/api/v1/query" + queryRangePath = "/loki/api/v1/query_range" + labelsPath = "/loki/api/v1/labels" + labelValuesPath = "/loki/api/v1/label/%s/values" + seriesPath = "/loki/api/v1/series" + tailPath = "/loki/api/v1/tail" + statsPath = "/loki/api/v1/index/stats" + volumePath = "/loki/api/v1/index/volume" + volumeRangePath = "/loki/api/v1/index/volume_range" + detectedFieldsPath = "/loki/api/v1/detected_fields" + defaultAuthHeader = "Authorization" ) var userAgent = fmt.Sprintf("loki-logcli/%s", build.Version) @@ -54,6 +55,7 @@ type Client interface { GetStats(queryStr string, start, end time.Time, quiet bool) (*logproto.IndexStatsResponse, error) GetVolume(query *volume.Query) (*loghttp.QueryResponse, error) GetVolumeRange(query *volume.Query) (*loghttp.QueryResponse, error) + GetDetectedFields(queryStr string, fieldLimit, lineLimit int, start, end time.Time, step time.Duration, quiet bool) (*loghttp.DetectedFieldsResponse, error) } // Tripperware can wrap a roundtripper. @@ -224,7 +226,36 @@ func (c *DefaultClient) getVolume(path string, query *volume.Query) (*loghttp.Qu return &resp, nil } -func (c *DefaultClient) doQuery(path string, query string, quiet bool) (*loghttp.QueryResponse, error) { +func (c *DefaultClient) GetDetectedFields( + queryStr string, + fieldLimit, lineLimit int, + start, end time.Time, + step time.Duration, + quiet bool, +) (*loghttp.DetectedFieldsResponse, error) { + qsb := util.NewQueryStringBuilder() + qsb.SetString("query", queryStr) + qsb.SetInt("field_limit", int64(fieldLimit)) + qsb.SetInt("line_limit", int64(lineLimit)) + qsb.SetInt("start", start.UnixNano()) + qsb.SetInt("end", end.UnixNano()) + qsb.SetString("step", step.String()) + + var err error + var r loghttp.DetectedFieldsResponse + + if err = c.doRequest(detectedFieldsPath, qsb.Encode(), quiet, &r); err != nil { + return nil, err + } + + return &r, nil +} + +func (c *DefaultClient) doQuery( + path string, + query string, + quiet bool, +) (*loghttp.QueryResponse, error) { var err error var r loghttp.QueryResponse diff --git a/pkg/logcli/client/file.go b/pkg/logcli/client/file.go index dd0432a79e17..34b76422d4bc 100644 --- a/pkg/logcli/client/file.go +++ b/pkg/logcli/client/file.go @@ -190,17 +190,28 @@ func (f *FileClient) GetOrgID() string { } func (f *FileClient) GetStats(_ string, _, _ time.Time, _ bool) (*logproto.IndexStatsResponse, error) { - // TODO(trevorwhitney): could we teach logcli to read from an actual index file? + // TODO(twhitney): could we teach logcli to read from an actual index file? return nil, ErrNotSupported } func (f *FileClient) GetVolume(_ *volume.Query) (*loghttp.QueryResponse, error) { - // TODO(trevorwhitney): could we teach logcli to read from an actual index file? + // TODO(twhitney): could we teach logcli to read from an actual index file? return nil, ErrNotSupported } func (f *FileClient) GetVolumeRange(_ *volume.Query) (*loghttp.QueryResponse, error) { - // TODO(trevorwhitney): could we teach logcli to read from an actual index file? + // TODO(twhitney): could we teach logcli to read from an actual index file? + return nil, ErrNotSupported +} + +func (f *FileClient) GetDetectedFields( + _ string, + _, _ int, + _, _ time.Time, + _ time.Duration, + _ bool, +) (*loghttp.DetectedFieldsResponse, error) { + // TODO(twhitney): could we teach logcli to do this? return nil, ErrNotSupported } diff --git a/pkg/logcli/detected/fields.go b/pkg/logcli/detected/fields.go new file mode 100644 index 000000000000..f8ba585ea2a0 --- /dev/null +++ b/pkg/logcli/detected/fields.go @@ -0,0 +1,57 @@ +package detected + +import ( + "encoding/json" + "fmt" + "log" + "slices" + "strings" + "time" + + "github.com/fatih/color" + + "github.com/grafana/loki/v3/pkg/logcli/client" + "github.com/grafana/loki/v3/pkg/loghttp" +) + +type FieldsQuery struct { + QueryString string + Start time.Time + End time.Time + FieldLimit int + LineLimit int + Step time.Duration + Quiet bool + ColoredOutput bool +} + +// DoQuery executes the query and prints out the results +func (q *FieldsQuery) Do(c client.Client, outputMode string) { + var resp *loghttp.DetectedFieldsResponse + var err error + + resp, err = c.GetDetectedFields(q.QueryString, q.FieldLimit, q.LineLimit, q.Start, q.End, q.Step, q.Quiet) + if err != nil { + log.Fatalf("Error doing request: %+v", err) + } + + switch outputMode { + case "raw": + out, err := json.Marshal(resp) + if err != nil { + log.Fatalf("Error marshalling response: %+v", err) + } + fmt.Println(string(out)) + default: + output := make([]string, len(resp.Fields)) + for i, field := range resp.Fields { + bold := color.New(color.Bold) + output[i] = fmt.Sprintf("label: %s\t\t", bold.Sprintf("%s", field.Label)) + + fmt.Sprintf("type: %s\t\t", bold.Sprintf("%s", field.Type)) + + fmt.Sprintf("cardinality: %s", bold.Sprintf("%d", field.Cardinality)) + } + + slices.Sort(output) + fmt.Println(strings.Join(output, "\n")) + } +} diff --git a/pkg/logcli/query/query_test.go b/pkg/logcli/query/query_test.go index c7543fa2288d..fffdcf15444b 100644 --- a/pkg/logcli/query/query_test.go +++ b/pkg/logcli/query/query_test.go @@ -485,6 +485,16 @@ func (t *testQueryClient) GetVolumeRange(_ *volume.Query) (*loghttp.QueryRespons panic("not implemented") } +func (t *testQueryClient) GetDetectedFields( + _ string, + _, _ int, + _, _ time.Time, + _ time.Duration, + _ bool, +) (*loghttp.DetectedFieldsResponse, error) { + panic("not implemented") +} + var legacySchemaConfigContents = `schema_config: configs: - from: 2020-05-15 diff --git a/pkg/loghttp/detected.go b/pkg/loghttp/detected.go new file mode 100644 index 000000000000..d255bf6124a7 --- /dev/null +++ b/pkg/loghttp/detected.go @@ -0,0 +1,14 @@ +package loghttp + +import "github.com/grafana/loki/v3/pkg/logproto" + +// LabelResponse represents the http json response to a label query +type DetectedFieldsResponse struct { + Fields []DetectedField `json:"fields,omitempty"` +} + +type DetectedField struct { + Label string `json:"label,omitempty"` + Type logproto.DetectedFieldType `json:"type,omitempty"` + Cardinality uint64 `json:"cardinality,omitempty"` +}