Skip to content

Commit b29ddfc

Browse files
kaiyan-shengkaiyan-sheng
authored
Fix mysql slowlog template to support mysql 5.7.22 (#9647)
* Never default to a qualifier when none of them are set. (#9148) Remove default version qualifier and rename the environment variable to set it from `BEAT_VERSION_QUALIFIER` to `VERSION_QUALIFIER` this will align with other parts of the stack. **Tested with filebeat.** ``` ❯ ./filebeat version [08:39:01] filebeat version 7.0.0 (amd64), libbeat 7.0.0 [0a0c267 built 2018-11-19 13:38:15 +0000 UTC] ``` **Without the patch** ``` ❯ ./filebeat version [08:40:07] filebeat version 7.0.0-alpha1 (amd64), libbeat 7.0.0-alpha1 [b007837 built 2018-11-19 13:39:59 +0000 UTC] ``` Fixes: #8384 * Fix mysql slowlog template to support mysql 5.7.22 * Add changelog * Remove changes introduced from rebasing that are not related to this change * Add more log entries to test log file * Add OR in multiline.pattern to remove # Time from mysql.slowlog.query * Use CHANGELOG.next.asciidoc instead
1 parent e5afe89 commit b29ddfc

File tree

4 files changed

+116
-2
lines changed

4 files changed

+116
-2
lines changed

‎CHANGELOG.next.asciidoc

+1
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
8585
- Added netflow input type that supports NetFlow v1, v5, v6, v7, v8, v9 and IPFIX. {issue}9399[9399]
8686
- Add option to modules.yml file to indicate that a module has been moved {pull}9432[9432].
8787
- Fix parsing of GC entries in elasticsearch server log. {issue}9513[9513] {pull}9810[9810]
88+
- Support mysql 5.7.22 slowlog starting with time information. {issue}7892[7892] {pull}9647[9647]
8889

8990
*Heartbeat*
9091

‎filebeat/module/mysql/slowlog/config/slowlog.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ paths:
55
{{ end }}
66
exclude_files: ['.gz$']
77
multiline:
8-
pattern: '^# User@Host: '
8+
pattern: '^(# User@Host: |# Time: )'
99
negate: true
1010
match: after
11-
exclude_lines: ['^[\/\w\.]+, Version: .* started with:.*'] # Exclude the header
11+
exclude_lines: ['^[\/\w\.]+, Version: .* started with:.*', '^# Time:.*'] # Exclude the header and time

‎filebeat/module/mysql/slowlog/test/mysql-5.7.22.log

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Time: 2018-08-07T16:27:47.169604+08:00
2+
# User@Host: root[root] @ [218.76.8.37] Id: 7234
3+
# Query_time: 15.000223 Lock_time: 0.000000 Rows_sent: 1 Rows_examined: 0
4+
SET timestamp=1533630467;
5+
select sleep(15);
6+
# Time: 2018-08-07T16:27:47.169604+08:00
7+
# User@Host: debian-sys-maint[debian-sys-maint] @ localhost []
8+
# Query_time: 0.000153 Lock_time: 0.000061 Rows_sent: 1 Rows_examined: 5
9+
SET timestamp=1533630467;
10+
SELECT count(*) FROM mysql.user WHERE user='root' and password='';
11+
# Time: 2018-08-07T16:27:47.169604+08:00
12+
# User@Host: apphost[apphost] @ apphost [1.1.1.1] Id: 10997316
13+
# Query_time: 4.071491 Lock_time: 0.000212 Rows_sent: 1000 Rows_examined: 1489615
14+
SET timestamp=1533630467;
15+
SELECT mcu.mcu_guid, mcu.cus_guid, mcu.mcu_url, mcu.mcu_crawlelements, mcu.mcu_order, GROUP_CONCAT(mca.mca_guid SEPARATOR ";") as mca_guid
16+
FROM kat_mailcustomerurl mcu, kat_customer cus, kat_mailcampaign mca
17+
WHERE cus.cus_guid = mcu.cus_guid
18+
AND cus.pro_code = 'CYB'
19+
AND cus.cus_offline = 0
20+
AND mca.cus_guid = cus.cus_guid
21+
AND (mcu.mcu_date IS NULL OR mcu.mcu_date < CURDATE())
22+
AND mcu.mcu_crawlelements IS NOT NULL
23+
GROUP BY mcu.mcu_guid
24+
ORDER BY mcu.mcu_order ASC
25+
LIMIT 1000;
26+
# Time: 2018-08-07T16:27:47.169604+08:00
27+
# User@Host: apphost[apphost] @ apphost [1.1.1.1] Id: 10999834
28+
# Query_time: 10.346539 Lock_time: 0.000036 Rows_sent: 0 Rows_examined: 4751313
29+
SET timestamp=1533630467;
30+
call load_stats(1, '2017-04-28 00:00:00');

‎filebeat/module/mysql/slowlog/test/mysql-5.7.22.log-expected.json

+83
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
[
2+
{
3+
"@timestamp": "2018-08-07T08:27:47.000Z",
4+
"ecs.version": "1.0.0-beta2",
5+
"event.dataset": "slowlog",
6+
"event.module": "mysql",
7+
"input.type": "log",
8+
"log.flags": [
9+
"multiline"
10+
],
11+
"log.offset": 41,
12+
"mysql.slowlog.id": "7234",
13+
"mysql.slowlog.ip": "218.76.8.37",
14+
"mysql.slowlog.lock_time.sec": "0.000000",
15+
"mysql.slowlog.query": "select sleep(15);",
16+
"mysql.slowlog.query_time.sec": "15.000223",
17+
"mysql.slowlog.rows_examined": "0",
18+
"mysql.slowlog.rows_sent": "1",
19+
"mysql.slowlog.timestamp": "1533630467",
20+
"mysql.slowlog.user": "root"
21+
},
22+
{
23+
"@timestamp": "2018-08-07T08:27:47.000Z",
24+
"ecs.version": "1.0.0-beta2",
25+
"event.dataset": "slowlog",
26+
"event.module": "mysql",
27+
"input.type": "log",
28+
"log.flags": [
29+
"multiline"
30+
],
31+
"log.offset": 254,
32+
"mysql.slowlog.host": "localhost",
33+
"mysql.slowlog.lock_time.sec": "0.000061",
34+
"mysql.slowlog.query": "SELECT count(*) FROM mysql.user WHERE user='root' and password='';",
35+
"mysql.slowlog.query_time.sec": "0.000153",
36+
"mysql.slowlog.rows_examined": "5",
37+
"mysql.slowlog.rows_sent": "1",
38+
"mysql.slowlog.timestamp": "1533630467",
39+
"mysql.slowlog.user": "debian-sys-maint"
40+
},
41+
{
42+
"@timestamp": "2018-08-07T08:27:47.000Z",
43+
"ecs.version": "1.0.0-beta2",
44+
"event.dataset": "slowlog",
45+
"event.module": "mysql",
46+
"input.type": "log",
47+
"log.flags": [
48+
"multiline"
49+
],
50+
"log.offset": 526,
51+
"mysql.slowlog.host": "apphost",
52+
"mysql.slowlog.id": "10997316",
53+
"mysql.slowlog.ip": "1.1.1.1",
54+
"mysql.slowlog.lock_time.sec": "0.000212",
55+
"mysql.slowlog.query": "SELECT mcu.mcu_guid, mcu.cus_guid, mcu.mcu_url, mcu.mcu_crawlelements, mcu.mcu_order, GROUP_CONCAT(mca.mca_guid SEPARATOR \";\") as mca_guid\n FROM kat_mailcustomerurl mcu, kat_customer cus, kat_mailcampaign mca\n WHERE cus.cus_guid = mcu.cus_guid\n AND cus.pro_code = 'CYB'\n AND cus.cus_offline = 0\n AND mca.cus_guid = cus.cus_guid\n AND (mcu.mcu_date IS NULL OR mcu.mcu_date < CURDATE())\n AND mcu.mcu_crawlelements IS NOT NULL\n GROUP BY mcu.mcu_guid\n ORDER BY mcu.mcu_order ASC\n LIMIT 1000;",
56+
"mysql.slowlog.query_time.sec": "4.071491",
57+
"mysql.slowlog.rows_examined": "1489615",
58+
"mysql.slowlog.rows_sent": "1000",
59+
"mysql.slowlog.timestamp": "1533630467",
60+
"mysql.slowlog.user": "apphost"
61+
},
62+
{
63+
"@timestamp": "2018-08-07T08:27:47.000Z",
64+
"ecs.version": "1.0.0-beta2",
65+
"event.dataset": "slowlog",
66+
"event.module": "mysql",
67+
"input.type": "log",
68+
"log.flags": [
69+
"multiline"
70+
],
71+
"log.offset": 1438,
72+
"mysql.slowlog.host": "apphost",
73+
"mysql.slowlog.id": "10999834",
74+
"mysql.slowlog.ip": "1.1.1.1",
75+
"mysql.slowlog.lock_time.sec": "0.000036",
76+
"mysql.slowlog.query": "call load_stats(1, '2017-04-28 00:00:00');",
77+
"mysql.slowlog.query_time.sec": "10.346539",
78+
"mysql.slowlog.rows_examined": "4751313",
79+
"mysql.slowlog.rows_sent": "0",
80+
"mysql.slowlog.timestamp": "1533630467",
81+
"mysql.slowlog.user": "apphost"
82+
}
83+
]

0 commit comments

Comments
 (0)