-
Notifications
You must be signed in to change notification settings - Fork 137
Closed
Labels
Description
Describe the bug
If I use the example below I get the data I expected.
If I add appname or any other String based item I'll get an InvalidUtf8Encoding(Utf8Error { valid_up_to: 84, error_len: Some(1) })
I don't know why the error occurs, because the data below seems good to me. (I exported the Data with DataGrip, maybe the bad encoding gets cleared?)
Code example
#[derive(Debug, Serialize, Deserialize, Row)]
pub struct SyslogBsdLog {
#[serde(with = "clickhouse::serde::uuid")]
log_id: uuid::Uuid,
#[serde(with = "clickhouse::serde::uuid")]
project_id: uuid::Uuid,
// appname: String,
// facility: String,
// hostname: String,
// message: String,
// severity: String,
#[serde(with = "clickhouse::serde::ipv4")]
source_ip: std::net::Ipv4Addr,
// source_type: String,
// #[serde(with = "clickhouse::serde::time::date")]
// ts: String,
}
let syslog_bsd_logs: Vec<SyslogBsdLog> = ch_client
.query(
"SELECT ?fields, toUnixTimestamp(timestamp) AS ts FROM syslog_bsd_logs WHERE project_id = ? ORDER by ts DESC LIMIT ?",
)
.bind(req.project_id)
.bind(10)
.fetch_all()
.await
.map_err(|e| {
eprintln!("Error fetching data: {:?}", e);
tonic::Status::internal("Failed to fetch data")
})?;Error log
Query log
2024.11.05 23:59:17.587785 [ 67 ] {} <Trace> HTTPHandler-factory: HTTP Request for HTTPHandler-factory. Method: GET, Address: 192.168.50.1:38296, User-Agent: clickhouse-rs/0.13.1 (lv:rust/1.70.0, os:linux), Length: 0, Content Type: , Transfer Encoding: identity, X-Forwarded-For: (none)
2024.11.05 23:59:17.587870 [ 67 ] {} <Trace> DynamicQueryHandler: Request URI: /?database=monitoring&query=SELECT+%60log_id%60%2C%60project_id%60%2C%60appname%60%2C%60facility%60%2C%60hostname%60%2C%60message%60%2C%60severity%60%2C%60source_ip%60%2C+trim%28source_type%29+AS+source_type%2C+toUnixTimestamp%28timestamp%29+AS+ts+FROM+syslog_bsd_logs+WHERE+project_id+%3D+%27cd61874b-d688-4e7e-a307-6ade37cebb84%27+ORDER+by+ts+DESC+LIMIT+10+FORMAT+RowBinary&compress=1
2024.11.05 23:59:17.587893 [ 67 ] {} <Debug> HTTP-Session: 04facb52-06f9-4647-a103-621565b72a4f Authenticating user 'default' from 192.168.50.1:38296
2024.11.05 23:59:17.587929 [ 67 ] {} <Debug> HTTP-Session: 04facb52-06f9-4647-a103-621565b72a4f Authenticated with global context as user 94309d50-4f52-5250-31bd-74fecac179db
2024.11.05 23:59:17.587940 [ 67 ] {} <Debug> HTTP-Session: 04facb52-06f9-4647-a103-621565b72a4f Creating session context with user_id: 94309d50-4f52-5250-31bd-74fecac179db
2024.11.05 23:59:17.588026 [ 67 ] {} <Trace> HTTP-Session: 04facb52-06f9-4647-a103-621565b72a4f Creating query context from session context, user_id: 94309d50-4f52-5250-31bd-74fecac179db, parent context user: default
2024.11.05 23:59:17.588321 [ 67 ] {b71590a3-a0ee-4038-8fdc-f7714421f439} <Debug> executeQuery: (from 192.168.50.1:38296) SELECT `log_id`,`project_id`,`appname`,`facility`,`hostname`,`message`,`severity`,`source_ip`, toUnixTimestamp(timestamp) AS ts FROM syslog_bsd_logs WHERE project_id = 'cd61874b-d688-4e7e-a307-6ade37cebb84' ORDER by ts DESC LIMIT 10 FORMAT RowBinary (stage: Complete)
2024.11.05 23:59:17.588695 [ 67 ] {b71590a3-a0ee-4038-8fdc-f7714421f439} <Trace> Planner: Query to stage Complete
2024.11.05 23:59:17.588806 [ 67 ] {b71590a3-a0ee-4038-8fdc-f7714421f439} <Trace> Planner: Query from stage FetchColumns to stage Complete
2024.11.05 23:59:17.589259 [ 67 ] {b71590a3-a0ee-4038-8fdc-f7714421f439} <Trace> QueryPlanOptimizePrewhere: The min valid primary key position for moving to the tail of PREWHERE is -1
2024.11.05 23:59:17.589279 [ 67 ] {b71590a3-a0ee-4038-8fdc-f7714421f439} <Trace> QueryPlanOptimizePrewhere: Condition equals(project_id, 'cd61874b-d688-4e7e-a307-6ade37cebb84'_String) moved to PREWHERE
2024.11.05 23:59:17.589359 [ 67 ] {b71590a3-a0ee-4038-8fdc-f7714421f439} <Debug> DATABASE.syslog_bsd_logs (10cb3013-9fc3-4319-a532-df70c7639242) (SelectExecutor): Key condition: (column 1 in ['cd61874b-d688-4e7e-a307-6ade37cebb84', 'cd61874b-d688-4e7e-a307-6ade37cebb84'])
2024.11.05 23:59:17.589380 [ 67 ] {b71590a3-a0ee-4038-8fdc-f7714421f439} <Trace> DATABASE.syslog_bsd_logs (10cb3013-9fc3-4319-a532-df70c7639242) (SelectExecutor): Filtering marks by primary and secondary keys
2024.11.05 23:59:17.591017 [ 709 ] {b71590a3-a0ee-4038-8fdc-f7714421f439} <Trace> DATABASE.syslog_bsd_logs (10cb3013-9fc3-4319-a532-df70c7639242) (SelectExecutor): Used generic exclusion search over index for part all_1_3877_2379 with 1 steps
2024.11.05 23:59:17.591059 [ 752 ] {b71590a3-a0ee-4038-8fdc-f7714421f439} <Trace> DATABASE.syslog_bsd_logs (10cb3013-9fc3-4319-a532-df70c7639242) (SelectExecutor): Used generic exclusion search over index for part all_3878_3878_0 with 1 steps
2024.11.05 23:59:17.591091 [ 709 ] {b71590a3-a0ee-4038-8fdc-f7714421f439} <Trace> DATABASE.syslog_bsd_logs (10cb3013-9fc3-4319-a532-df70c7639242) (SelectExecutor): Used generic exclusion search over index for part all_3879_3879_0 with 1 steps
2024.11.05 23:59:17.591155 [ 709 ] {b71590a3-a0ee-4038-8fdc-f7714421f439} <Trace> DATABASE.syslog_bsd_logs (10cb3013-9fc3-4319-a532-df70c7639242) (SelectExecutor): Used generic exclusion search over index for part all_3880_3880_0 with 1 steps
2024.11.05 23:59:17.591220 [ 709 ] {b71590a3-a0ee-4038-8fdc-f7714421f439} <Trace> DATABASE.syslog_bsd_logs (10cb3013-9fc3-4319-a532-df70c7639242) (SelectExecutor): Used generic exclusion search over index for part all_3881_3881_0 with 1 steps
2024.11.05 23:59:17.591392 [ 67 ] {b71590a3-a0ee-4038-8fdc-f7714421f439} <Debug> DATABASE.syslog_bsd_logs (10cb3013-9fc3-4319-a532-df70c7639242) (SelectExecutor): Selected 5/5 parts by partition key, 5 parts by primary key, 5/5 marks by primary key, 5 marks to read from 5 ranges
2024.11.05 23:59:17.591412 [ 67 ] {b71590a3-a0ee-4038-8fdc-f7714421f439} <Trace> DATABASE.syslog_bsd_logs (10cb3013-9fc3-4319-a532-df70c7639242) (SelectExecutor): Spreading mark ranges among streams (default reading)
2024.11.05 23:59:17.591532 [ 67 ] {b71590a3-a0ee-4038-8fdc-f7714421f439} <Debug> DATABASE.syslog_bsd_logs (10cb3013-9fc3-4319-a532-df70c7639242) (SelectExecutor): Reading approx. 5077 rows with 2 streams
2024.11.05 23:59:17.591561 [ 67 ] {b71590a3-a0ee-4038-8fdc-f7714421f439} <Trace> MergeTreeSelectProcessor: PREWHERE condition was split into 1 steps: "equals(__table1.project_id, 'cd61874b-d688-4e7e-a307-6ade37cebb84'_String)"
2024.11.05 23:59:17.591611 [ 67 ] {b71590a3-a0ee-4038-8fdc-f7714421f439} <Trace> MergeTreeSelectProcessor: PREWHERE condition was split into 1 steps: "equals(__table1.project_id, 'cd61874b-d688-4e7e-a307-6ade37cebb84'_String)"
2024.11.05 23:59:17.595178 [ 67 ] {b71590a3-a0ee-4038-8fdc-f7714421f439} <Debug> executeQuery: Read 5077 rows, 1.24 MiB in 0.006992 sec., 726115.5606407323 rows/sec., 177.96 MiB/sec.
2024.11.05 23:59:17.595892 [ 67 ] {b71590a3-a0ee-4038-8fdc-f7714421f439} <Debug> DynamicQueryHandler: Done processing query
2024.11.05 23:59:17.595915 [ 67 ] {b71590a3-a0ee-4038-8fdc-f7714421f439} <Debug> MemoryTracker: Peak memory usage (for query): 4.02 MiB.
2024.11.05 23:59:17.595934 [ 67 ] {} <Debug> HTTP-Session: 04facb52-06f9-4647-a103-621565b72a4f Logout, user_id: 94309d50-4f52-5250-31bd-74fecac179db
Configuration
Environment
- Client version:
clickhouse = { version = "0.13.1", features = ["uuid", "time"] } - OS: Linux Fedora 41
ClickHouse server
- ClickHouse Server version: 24.9.2.42
- ClickHouse Server non-default settings, if any:
CREATE TABLEstatements for tables involved:
CREATE TABLE IF NOT EXISTS DATABASE.syslog_bsd_logs (
log_id String,
project_id String,
appname String,
facility String,
hostname String,
message String,
severity String,
source_ip String,
source_type String,
timestamp DateTime('Europe/Berlin')
)
ENGINE = MergeTree()
PRIMARY KEY (log_id, project_id, hostname, timestamp)- Sample data for all these tables, use [clickhouse-obfuscator]
INSERT INTO syslog_bsd_logs (log_id, project_id, appname, facility, hostname, message, severity, source_ip, source_type, timestamp) VALUES ('56cde52f-5f34-45e0-9f08-79d6f582e913', 'cd61874b-d688-4e7e-a307-6ade37cebb84', 'dhcp_private', 'daemon', 'router-1', 'assigned 192.168.40.49 for [MAC]', 'info', '192.168.50.1', 'syslog', 2024-11-05T11:52:52+01:00);
INSERT INTO syslog_bsd_logs (log_id, project_id, appname, facility, hostname, message, severity, source_ip, source_type, timestamp) VALUES ('0e967129-6271-44f2-967b-0c8d11a60fdc', 'cd61874b-d688-4e7e-a307-6ade37cebb84', 'EC', 'daemon', 'router-1', '[MAC]@cap-wifi2-virtual1 connected, signal strength -26', 'info', '192.168.50.1', 'syslog', 2024-11-05T11:59:21+01:00);
INSERT INTO syslog_bsd_logs (log_id, project_id, appname, facility, hostname, message, severity, source_ip, source_type, timestamp) VALUES ('888bd0da-649e-4d06-9730-84b19f5262dd', 'cd61874b-d688-4e7e-a307-6ade37cebb84', 'EC', 'daemon', 'router-1', '[MAC]@cap-wifi2-virtual1 disconnected, connection lost, signal strength -20', 'info', '192.168.50.1', 'syslog', 2024-11-05T11:59:20+01:00);
INSERT INTO syslog_bsd_logs (log_id, project_id, appname, facility, hostname, message, severity, source_ip, source_type, timestamp) VALUES ('5a9514b9-5ecc-449e-8a7c-79f6c14f05b6', 'cd61874b-d688-4e7e-a307-6ade37cebb84', 'dhcp_guest', 'daemon', 'router-1', 'assigned 192.168.200.149 for [MAC]', 'info', '192.168.50.1', 'syslog', 2024-11-05T11:59:21+01:00);
INSERT INTO syslog_bsd_logs (log_id, project_id, appname, facility, hostname, message, severity, source_ip, source_type, timestamp) VALUES ('e482d6b3-911f-43cd-920b-234fa207b1c3', 'cd61874b-d688-4e7e-a307-6ade37cebb84', 'denied', 'daemon', 'router-1', 'winbox/dude connect from 118.123.105.89', 'info', '192.168.50.1', 'syslog', 2024-11-05T11:42:25+01:00);
INSERT INTO syslog_bsd_logs (log_id, project_id, appname, facility, hostname, message, severity, source_ip, source_type, timestamp) VALUES ('001d030b-98c8-4e04-880d-2d2963a2c3fe', 'cd61874b-d688-4e7e-a307-6ade37cebb84', 'Detected', 'daemon', 'router-1', 'conflict by ARP response for 192.168.200.173 from [MAC]', 'emerg', '192.168.50.1', 'syslog', 2024-10-27T08:54:27+01:00);
INSERT INTO syslog_bsd_logs (log_id, project_id, appname, facility, hostname, message, severity, source_ip, source_type, timestamp) VALUES ('001f336e-d9b1-4551-aa79-e539d4a4d818', 'cd61874b-d688-4e7e-a307-6ade37cebb84', 'denied', 'daemon', 'router-1', 'winbox/dude connect from **.***.***.**', 'emerg', '192.168.50.1', 'syslog', 2024-10-28T20:07:59+01:00);
INSERT INTO syslog_bsd_logs (log_id, project_id, appname, facility, hostname, message, severity, source_ip, source_type, timestamp) VALUES ('002802f3-a4b6-4829-8d1d-eb9caec989d4', 'cd61874b-d688-4e7e-a307-6ade37cebb84', 'dhcp_private', 'daemon', 'router-1', 'assigned 192.168.40.197 for [MAC] Nuki_Bridge_*****', 'emerg', '192.168.50.1', 'syslog', 2024-10-30T09:57:18+01:00);
INSERT INTO syslog_bsd_logs (log_id, project_id, appname, facility, hostname, message, severity, source_ip, source_type, timestamp) VALUES ('0049463f-2224-405f-ac9c-e4514301fa96', 'cd61874b-d688-4e7e-a307-6ade37cebb84', 'EC', 'daemon', 'router-1', '[MAC]@cap-wifi3-virtual1 disconnected, connection lost, signal strength -79', 'emerg', '192.168.50.1', 'syslog', 2024-10-23T00:21+02:00);
INSERT INTO syslog_bsd_logs (log_id, project_id, appname, facility, hostname, message, severity, source_ip, source_type, timestamp) VALUES ('004cd65b-9b6b-4e78-8fe4-c45d98342b17', 'cd61874b-d688-4e7e-a307-6ade37cebb84', 'denied', 'daemon', 'router-1', 'winbox/dude connect from **.***.***.**', 'emerg', '192.168.50.1', 'syslog', 2024-10-28T20:07:29+01:00);
INSERT INTO syslog_bsd_logs (log_id, project_id, appname, facility, hostname, message, severity, source_ip, source_type, timestamp) VALUES ('005efe83-5ea6-4b60-b8c6-7eef7ccbef8d', 'cd61874b-d688-4e7e-a307-6ade37cebb84', 'denied', 'daemon', 'router-1', 'winbox/dude connect from **.***.***.**', 'emerg', '192.168.50.1', 'syslog', 2024-10-28T20:08:49+01:00);
INSERT INTO syslog_bsd_logs (log_id, project_id, appname, facility, hostname, message, severity, source_ip, source_type, timestamp) VALUES ('0061e53a-7081-428d-9dab-e39347c264c3', 'cd61874b-d688-4e7e-a307-6ade37cebb84', 'B6', 'daemon', 'router-1', '[MAC]@cap-wifi4 connected, signal strength -44', 'emerg', '192.168.50.1', 'syslog', 2024-10-20T21:56:32+02:00);
INSERT INTO syslog_bsd_logs (log_id, project_id, appname, facility, hostname, message, severity, source_ip, source_type, timestamp) VALUES ('00674d7e-5b3a-4cb1-99fb-ef497763a412', 'cd61874b-d688-4e7e-a307-6ade37cebb84', '3C', 'daemon', 'router-1', '[MAC]@cap-wifi1 disconnected, SA Query timeout, signal strength -80', 'emerg', '192.168.50.1', 'syslog', 2024-10-26T08:54:36+02:00);
INSERT INTO syslog_bsd_logs (log_id, project_id, appname, facility, hostname, message, severity, source_ip, source_type, timestamp) VALUES ('0078fe72-bde9-4baf-9fdb-c3d33fb2b28f', 'cd61874b-d688-4e7e-a307-6ade37cebb84', 'EC', 'daemon', 'router-1', '[MAC]@cap-wifi3-virtual1 connected, signal strength -77', 'emerg', '192.168.50.1', 'syslog', 2024-10-23T04:48:36+02:00);
INSERT INTO syslog_bsd_logs (log_id, project_id, appname, facility, hostname, message, severity, source_ip, source_type, timestamp) VALUES ('008bfe34-e5fd-42a6-a26b-ffb365899fe7', 'cd61874b-d688-4e7e-a307-6ade37cebb84', 'dhcp_guest', 'daemon', 'router-1', 'assigned 192.168.200.74 for [MAC]', 'emerg', '192.168.50.1', 'syslog', 2024-11-02T08:59:21+01:00);
INSERT INTO syslog_bsd_logs (log_id, project_id, appname, facility, hostname, message, severity, source_ip, source_type, timestamp) VALUES ('008c0f98-768b-4e0e-907a-08741aed198c', 'cd61874b-d688-4e7e-a307-6ade37cebb84', 'dhcp_guest', 'daemon', 'router-1', 'deassigned 192.168.200.77 for [MAC]', 'emerg', '192.168.50.1', 'syslog', 2024-10-27T13:47:56+01:00);
INSERT INTO syslog_bsd_logs (log_id, project_id, appname, facility, hostname, message, severity, source_ip, source_type, timestamp) VALUES ('0095e89e-e67c-48e3-9bdd-ef5d36bb0672', 'cd61874b-d688-4e7e-a307-6ade37cebb84', 'dhcp_guest', 'daemon', 'router-1', 'assigned 192.168.200.133 for [MAC]', 'emerg', '192.168.50.1', 'syslog', 2024-10-25T14:20:09+02:00);