Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[exporter/clickhouse] Update default logs table schema #33611

Merged
merged 3 commits into from
Jun 19, 2024
Merged

[exporter/clickhouse] Update default logs table schema #33611

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0c4125d
update logs schema
SpencerTorres Jun 18, 2024
bba1a2e
update changelog
SpencerTorres Jun 18, 2024
168d838
add ServiceName to ORDER BY
SpencerTorres Jun 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions .chloggen/clickhouseexporter_update_default_logs_table.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# Use this changelog template to create an entry for release notes.

# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
change_type: enhancement

# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
component: clickhouseexporter

# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
note: Updated the default logs table to a more optimized schema

# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
issues: [33611]

# (Optional) One or more lines of additional information to render under the primary note.
# These lines will be padded with 2 spaces and then inserted directly into the document.
# Use pipe (|) for multiline entries.
subtext: Simplified data types, improved partitioning and time range queries.

# If your change doesn't affect end users or the exported elements of any package,
# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
# Optional: The change log or logs in which this entry should be included.
# e.g. '[user]' or '[user, api]'
# Include 'user' if the change is relevant to end users.
# Include 'api' if there is a change to a library API.
# Default: '[user]'
change_logs: []
16 changes: 8 additions & 8 deletions exporter/clickhouseexporter/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ as [ClickHouse document says:](https://clickhouse.com/docs/en/introduction/perfo
- Get log severity count time series.

```clickhouse
SELECT toDateTime(toStartOfInterval(Timestamp, INTERVAL 60 second)) as time, SeverityText, count() as count
SELECT toDateTime(toStartOfInterval(TimestampTime, INTERVAL 60 second)) as time, SeverityText, count() as count
FROM otel_logs
WHERE time >= NOW() - INTERVAL 1 HOUR
GROUP BY SeverityText, time
Expand All @@ -55,7 +55,7 @@ ORDER BY time;
```clickhouse
SELECT Timestamp as log_time, Body
FROM otel_logs
WHERE Timestamp >= NOW() - INTERVAL 1 HOUR
WHERE TimestampTime >= NOW() - INTERVAL 1 HOUR
Limit 100;
```

Expand All @@ -65,7 +65,7 @@ Limit 100;
SELECT Timestamp as log_time, Body
FROM otel_logs
WHERE ServiceName = 'clickhouse-exporter'
AND Timestamp >= NOW() - INTERVAL 1 HOUR
AND TimestampTime >= NOW() - INTERVAL 1 HOUR
Limit 100;
```

Expand All @@ -75,7 +75,7 @@ Limit 100;
SELECT Timestamp as log_time, Body
FROM otel_logs
WHERE LogAttributes['container_name'] = '/example_flog_1'
AND Timestamp >= NOW() - INTERVAL 1 HOUR
AND TimestampTime >= NOW() - INTERVAL 1 HOUR
Limit 100;
```

Expand All @@ -85,7 +85,7 @@ Limit 100;
SELECT Timestamp as log_time, Body
FROM otel_logs
WHERE hasToken(Body, 'http')
AND Timestamp >= NOW() - INTERVAL 1 HOUR
AND TimestampTime >= NOW() - INTERVAL 1 HOUR
Limit 100;
```

Expand All @@ -95,7 +95,7 @@ Limit 100;
SELECT Timestamp as log_time, Body
FROM otel_logs
WHERE Body like '%http%'
AND Timestamp >= NOW() - INTERVAL 1 HOUR
AND TimestampTime >= NOW() - INTERVAL 1 HOUR
Limit 100;
```

Expand All @@ -105,7 +105,7 @@ Limit 100;
SELECT Timestamp as log_time, Body
FROM otel_logs
WHERE match(Body, 'http')
AND Timestamp >= NOW() - INTERVAL 1 HOUR
AND TimestampTime >= NOW() - INTERVAL 1 HOUR
Limit 100;
```

Expand All @@ -115,7 +115,7 @@ Limit 100;
SELECT Timestamp as log_time, Body
FROM otel_logs
WHERE JSONExtractFloat(Body, 'bytes') > 1000
AND Timestamp >= NOW() - INTERVAL 1 HOUR
AND TimestampTime >= NOW() - INTERVAL 1 HOUR
Limit 100;
```

Expand Down
23 changes: 13 additions & 10 deletions exporter/clickhouseexporter/example/default_ddl/logs.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,21 +1,24 @@
-- Default Logs table DDL

CREATE TABLE IF NOT EXISTS otel_logs (
Timestamp DateTime64(9) CODEC(Delta, ZSTD(1)),
Timestamp DateTime64(9) CODEC(Delta(8), ZSTD(1)),
TimestampDate Date DEFAULT toDate(Timestamp),
TimestampTime DateTime DEFAULT toDateTime(Timestamp),
TraceId String CODEC(ZSTD(1)),
SpanId String CODEC(ZSTD(1)),
TraceFlags UInt32 CODEC(ZSTD(1)),
TraceFlags UInt8,
SeverityText LowCardinality(String) CODEC(ZSTD(1)),
SeverityNumber Int32 CODEC(ZSTD(1)),
SeverityNumber UInt8,
ServiceName LowCardinality(String) CODEC(ZSTD(1)),
Body String CODEC(ZSTD(1)),
ResourceSchemaUrl String CODEC(ZSTD(1)),
ResourceSchemaUrl LowCardinality(String) CODEC(ZSTD(1)),
ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
ScopeSchemaUrl String CODEC(ZSTD(1)),
ScopeSchemaUrl LowCardinality(String) CODEC(ZSTD(1)),
ScopeName String CODEC(ZSTD(1)),
ScopeVersion String CODEC(ZSTD(1)),
ScopeVersion LowCardinality(String) CODEC(ZSTD(1)),
ScopeAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
LogAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),

INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
Expand All @@ -25,7 +28,7 @@ CREATE TABLE IF NOT EXISTS otel_logs (
INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_body Body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1
) ENGINE = MergeTree()
TTL toDateTime("Timestamp") + toIntervalDay(180)
PARTITION BY toDate(Timestamp)
ORDER BY (ServiceName, SeverityText, toUnixTimestamp(Timestamp), TraceId)
SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;
PARTITION BY toYYYYMM(TimestampDate)
ORDER BY (TimestampDate, TimestampTime)
TTL TimestampTime + toIntervalDay(180)
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
57 changes: 30 additions & 27 deletions exporter/clickhouseexporter/exporter_logs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,34 +133,37 @@ const (
// language=ClickHouse SQL
createLogsTableSQL = `
CREATE TABLE IF NOT EXISTS %s %s (
Timestamp DateTime64(9) CODEC(Delta, ZSTD(1)),
TraceId String CODEC(ZSTD(1)),
SpanId String CODEC(ZSTD(1)),
TraceFlags UInt32 CODEC(ZSTD(1)),
SeverityText LowCardinality(String) CODEC(ZSTD(1)),
SeverityNumber Int32 CODEC(ZSTD(1)),
ServiceName LowCardinality(String) CODEC(ZSTD(1)),
Body String CODEC(ZSTD(1)),
ResourceSchemaUrl String CODEC(ZSTD(1)),
ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
ScopeSchemaUrl String CODEC(ZSTD(1)),
ScopeName String CODEC(ZSTD(1)),
ScopeVersion String CODEC(ZSTD(1)),
ScopeAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
LogAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_body Body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1
Timestamp DateTime64(9) CODEC(Delta(8), ZSTD(1)),
TimestampDate Date DEFAULT toDate(Timestamp),
TimestampTime DateTime DEFAULT toDateTime(Timestamp),
TraceId String CODEC(ZSTD(1)),
SpanId String CODEC(ZSTD(1)),
TraceFlags UInt8,
SeverityText LowCardinality(String) CODEC(ZSTD(1)),
SeverityNumber UInt8,
ServiceName LowCardinality(String) CODEC(ZSTD(1)),
Body String CODEC(ZSTD(1)),
ResourceSchemaUrl LowCardinality(String) CODEC(ZSTD(1)),
ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
ScopeSchemaUrl LowCardinality(String) CODEC(ZSTD(1)),
ScopeName String CODEC(ZSTD(1)),
ScopeVersion LowCardinality(String) CODEC(ZSTD(1)),
ScopeAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
LogAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),

INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_body Body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1
) ENGINE = %s
PARTITION BY toYYYYMM(TimestampDate)
ORDER BY (TimestampDate, TimestampTime)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

general LGTM. just wondering ORDER BY (TimestampDate, TimestampTime) may not better than ORDER BY ServiceName, SeverityText, toUnixTimestamp(Timestamp), beacause ServiceName and SeverityText( also as well-known logLevel) is the most common filter when query service log, can you write more text explain why remove them? thanks.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It can be difficult to model a decent table that works for everyone (consider issues like #32215)

The idea behind this change is to simply make time range queries work as best they can. Every log has a timestamp, but not always a proper ServiceName or Severity*. Some OTel data isn't composed very well. SeverityText might not be as good as SeverityNumber, and if we try to add all of these to ORDER BY then it starts to lose its purpose as a primary index.

For those who will be using this exporter in production, the goal is to have them use this as a starting point, and then add the appropriate columns to the ORDER BY depending on their primary column. For example, look at how this blog post chooses to use PodName and Timestamp:

ORDER BY (PodName, Timestamp)

What we've found is that a lot of these values are not necessary in the order by, especially ones after the Timestamp* columns (such as TraceId in the current version). Most log queries are going to be in a tight range of time (5, 15, 30, 60 minutes) and these smaller ORDER BY clauses favor this. Again, you can always go a step further by adding your preferred column to the start of it, such as ServiceName, SeverityText, PodName, or some combination of these.

The goal is to have a good default that is easy for others to expand on for their own deployment, while also still giving a good default for those who don't bother to configure one. If you're at the scale where this schema becomes a problem, you would already be using your own schema.

Copy link
Member

@hanjm hanjm Jun 18, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ServiceName is a required in spec. https://opentelemetry.io/docs/specs/semconv/resource/#service. to say the least, ServiceName is needed in order by columns. see also as #31670

in PodName blog case , i think most common usage query pattern is PodName like xxx%, not PodName= xxx, mostly query a workload log, not the specific pod log. actually, use a serviceName as workload name will be more reasonaly.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is reasonable enough since it is a default schema, and if someone needs to change it they're free to do so for their deployment.

As you suggested, I have added ServiceName to the default schema.

%s
PARTITION BY toDate(Timestamp)
ORDER BY (ServiceName, SeverityText, toUnixTimestamp(Timestamp), TraceId)
SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
`
// language=ClickHouse SQL
insertLogsSQLTemplate = `INSERT INTO %s (
Expand Down Expand Up @@ -238,7 +241,7 @@ func createLogsTable(ctx context.Context, cfg *Config, db *sql.DB) error {
}

func renderCreateLogsTableSQL(cfg *Config) string {
ttlExpr := generateTTLExpr(cfg.TTLDays, cfg.TTL, "Timestamp")
ttlExpr := generateTTLExpr(cfg.TTLDays, cfg.TTL, "TimestampTime")
return fmt.Sprintf(createLogsTableSQL, cfg.LogsTableName, cfg.ClusterString(), cfg.TableEngineString(), ttlExpr)
}

Expand Down
2 changes: 1 addition & 1 deletion exporter/clickhouseexporter/exporter_metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ func (e *metricsExporter) start(ctx context.Context, _ component.Host) error {
return err
}

ttlExpr := generateTTLExpr(e.cfg.TTLDays, e.cfg.TTL, "TimeUnix")
ttlExpr := generateTTLExpr(e.cfg.TTLDays, e.cfg.TTL, "toDateTime(TimeUnix)")
return internal.NewMetricsTable(ctx, e.cfg.MetricsTableName, e.cfg.ClusterString(), e.cfg.TableEngineString(), ttlExpr, e.client)
}

Expand Down
4 changes: 2 additions & 2 deletions exporter/clickhouseexporter/exporter_traces.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -295,12 +295,12 @@ func renderInsertTracesSQL(cfg *Config) string {
}

func renderCreateTracesTableSQL(cfg *Config) string {
ttlExpr := generateTTLExpr(cfg.TTLDays, cfg.TTL, "Timestamp")
ttlExpr := generateTTLExpr(cfg.TTLDays, cfg.TTL, "toDateTime(Timestamp)")
return fmt.Sprintf(createTracesTableSQL, cfg.TracesTableName, cfg.ClusterString(), cfg.TableEngineString(), ttlExpr)
}

func renderCreateTraceIDTsTableSQL(cfg *Config) string {
ttlExpr := generateTTLExpr(cfg.TTLDays, cfg.TTL, "Start")
ttlExpr := generateTTLExpr(cfg.TTLDays, cfg.TTL, "toDateTime(Start)")
return fmt.Sprintf(createTraceIDTsTableSQL, cfg.TracesTableName, cfg.ClusterString(), cfg.TableEngineString(), ttlExpr)
}

Expand Down
10 changes: 5 additions & 5 deletions exporter/clickhouseexporter/factory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,19 +126,19 @@ func createMetricExporter(

func generateTTLExpr(ttlDays uint, ttl time.Duration, timeField string) string {
if ttlDays > 0 {
return fmt.Sprintf(`TTL toDateTime(%s) + toIntervalDay(%d)`, timeField, ttlDays)
return fmt.Sprintf(`TTL %s + toIntervalDay(%d)`, timeField, ttlDays)
}

if ttl > 0 {
switch {
case ttl%(24*time.Hour) == 0:
return fmt.Sprintf(`TTL toDateTime(%s) + toIntervalDay(%d)`, timeField, ttl/(24*time.Hour))
return fmt.Sprintf(`TTL %s + toIntervalDay(%d)`, timeField, ttl/(24*time.Hour))
case ttl%(time.Hour) == 0:
return fmt.Sprintf(`TTL toDateTime(%s) + toIntervalHour(%d)`, timeField, ttl/time.Hour)
return fmt.Sprintf(`TTL %s + toIntervalHour(%d)`, timeField, ttl/time.Hour)
case ttl%(time.Minute) == 0:
return fmt.Sprintf(`TTL toDateTime(%s) + toIntervalMinute(%d)`, timeField, ttl/time.Minute)
return fmt.Sprintf(`TTL %s + toIntervalMinute(%d)`, timeField, ttl/time.Minute)
default:
return fmt.Sprintf(`TTL toDateTime(%s) + toIntervalSecond(%d)`, timeField, ttl/time.Second)
return fmt.Sprintf(`TTL %s + toIntervalSecond(%d)`, timeField, ttl/time.Second)
}
}
return ""
Expand Down
4 changes: 4 additions & 0 deletions exporter/clickhouseexporter/integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,8 @@ func verifyExportLog(t *testing.T, logExporter *logsExporter) {

type log struct {
Timestamp string `db:"Timestamp"`
TimestampDate string `db:"TimestampDate"`
TimestampTime string `db:"TimestampTime"`
TraceID string `db:"TraceId"`
SpanID string `db:"SpanId"`
TraceFlags uint32 `db:"TraceFlags"`
Expand All @@ -115,6 +117,8 @@ func verifyExportLog(t *testing.T, logExporter *logsExporter) {

expectLog := log{
Timestamp: "2023-12-25T09:53:49Z",
TimestampDate: "2023-12-25T00:00:00Z",
TimestampTime: "2023-12-25T09:53:49Z",
TraceID: "01020300000000000000000000000000",
SpanID: "0102030000000000",
SeverityText: "error",
Expand Down