Closed
Description
Looks like I have 2 sentinel instances with memory leak. From what I was able to collect, the memory increases indefinitely between 50kb and 100kb every second.
My setup:
- 3 machines
- each machine runs 12 redis-server instances and 1 redis-sentinel instance
- 12 masters and 24 slaves
About 2 days ago (on 16 sep), machine 2 died. As soon as it died, the memory consumption of both the remaining redis-sentinel (on machines 1 and 3) started rising.
Info output of both the instances:
[root@redis-lhv-01t ~]# redis-cli -h 172.30.104.126 -p 26379 info
# Server
redis_version:2.8.16
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:64e977e1ef8e7eba
redis_mode:sentinel
os:Linux 2.6.32-358.14.1.el6.x86_64 x86_64
arch_bits:64
multiplexing_api:epoll
gcc_version:4.4.7
process_id:56761
run_id:eeeeffe5c0c00f53f500bdfd40d2f64aaa453068
tcp_port:26379
uptime_in_seconds:8067
uptime_in_days:0
hz:14
lru_clock:1773952
config_file:/etc/redis/redis_sentinel.conf
# Sentinel
sentinel_masters:24
sentinel_tilt:0
sentinel_running_scripts:0
sentinel_scripts_queue_length:0
master0:name=Group_4,status=ok,address=172.30.104.126:60004,slaves=2,sentinels=3
master1:name=Group_23,status=ok,address=172.30.104.128:60007,slaves=2,sentinels=3
master2:name=Group_8,status=ok,address=172.30.104.128:61000,slaves=2,sentinels=3
master3:name=Group_14,status=ok,address=172.30.104.128:61006,slaves=2,sentinels=3
master4:name=Group_7,status=ok,address=172.30.104.126:60007,slaves=2,sentinels=3
master5:name=Group_2,status=ok,address=172.30.104.126:60002,slaves=2,sentinels=3
master6:name=Group_12,status=ok,address=172.30.104.128:61004,slaves=2,sentinels=3
master7:name=Group_9,status=ok,address=172.30.104.128:61001,slaves=2,sentinels=3
master8:name=Group_19,status=ok,address=172.30.104.128:60003,slaves=2,sentinels=3
master9:name=Group_13,status=ok,address=172.30.104.128:61005,slaves=2,sentinels=3
master10:name=Group_15,status=ok,address=172.30.104.128:61007,slaves=2,sentinels=3
master11:name=Group_21,status=ok,address=172.30.104.126:61005,slaves=2,sentinels=3
master12:name=Group_20,status=ok,address=172.30.104.128:60004,slaves=2,sentinels=3
master13:name=Group_10,status=ok,address=172.30.104.128:61002,slaves=2,sentinels=3
master14:name=Group_1,status=ok,address=172.30.104.126:60001,slaves=2,sentinels=3
master15:name=Group_18,status=ok,address=172.30.104.128:60002,slaves=2,sentinels=3
master16:name=Group_5,status=ok,address=172.30.104.126:60005,slaves=2,sentinels=3
master17:name=Group_17,status=ok,address=172.30.104.128:60001,slaves=2,sentinels=3
master18:name=Group_6,status=ok,address=172.30.104.126:60006,slaves=2,sentinels=3
master19:name=Group_3,status=ok,address=172.30.104.126:60003,slaves=2,sentinels=3
master20:name=Group_0,status=ok,address=172.30.104.126:60000,slaves=2,sentinels=3
master21:name=Group_16,status=ok,address=172.30.104.128:60000,slaves=2,sentinels=3
master22:name=Group_11,status=ok,address=172.30.104.128:61003,slaves=2,sentinels=3
master23:name=Group_22,status=ok,address=172.30.104.128:60006,slaves=2,sentinels=3
[root@redis-lhv-03t redis]# redis-cli -h 172.30.104.128 -p 26379 info
# Server
redis_version:2.8.16
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:ce5e684bcb0ba240
redis_mode:sentinel
os:Linux 2.6.32-358.14.1.el6.x86_64 x86_64
arch_bits:64
multiplexing_api:epoll
gcc_version:4.4.7
process_id:8646
run_id:cf51e9737a11ce1340025dece9093e8421568a8e
tcp_port:26379
uptime_in_seconds:8112
uptime_in_days:0
hz:18
lru_clock:1774167
config_file:/etc/redis/redis_sentinel.conf
# Sentinel
sentinel_masters:24
sentinel_tilt:0
sentinel_running_scripts:0
sentinel_scripts_queue_length:0
master0:name=Group_18,status=ok,address=172.30.104.128:60002,slaves=2,sentinels=3
master1:name=Group_2,status=ok,address=172.30.104.126:60002,slaves=2,sentinels=3
master2:name=Group_14,status=ok,address=172.30.104.128:61006,slaves=2,sentinels=3
master3:name=Group_0,status=ok,address=172.30.104.126:60000,slaves=2,sentinels=3
master4:name=Group_10,status=ok,address=172.30.104.128:61002,slaves=2,sentinels=3
master5:name=Group_8,status=ok,address=172.30.104.128:61000,slaves=2,sentinels=3
master6:name=Group_9,status=ok,address=172.30.104.128:61001,slaves=2,sentinels=3
master7:name=Group_20,status=ok,address=172.30.104.128:60004,slaves=2,sentinels=3
master8:name=Group_5,status=ok,address=172.30.104.126:60005,slaves=2,sentinels=3
master9:name=Group_12,status=ok,address=172.30.104.128:61004,slaves=2,sentinels=3
master10:name=Group_11,status=ok,address=172.30.104.128:61003,slaves=2,sentinels=3
master11:name=Group_22,status=ok,address=172.30.104.128:60006,slaves=2,sentinels=3
master12:name=Group_16,status=ok,address=172.30.104.128:60000,slaves=2,sentinels=3
master13:name=Group_13,status=ok,address=172.30.104.128:61005,slaves=2,sentinels=3
master14:name=Group_15,status=ok,address=172.30.104.128:61007,slaves=2,sentinels=3
master15:name=Group_3,status=ok,address=172.30.104.126:60003,slaves=2,sentinels=3
master16:name=Group_19,status=ok,address=172.30.104.128:60003,slaves=2,sentinels=3
master17:name=Group_17,status=ok,address=172.30.104.128:60001,slaves=2,sentinels=3
master18:name=Group_4,status=ok,address=172.30.104.126:60004,slaves=2,sentinels=3
master19:name=Group_21,status=ok,address=172.30.104.126:61005,slaves=2,sentinels=3
master20:name=Group_23,status=ok,address=172.30.104.128:60007,slaves=2,sentinels=3
master21:name=Group_1,status=ok,address=172.30.104.126:60001,slaves=2,sentinels=3
master22:name=Group_6,status=ok,address=172.30.104.126:60006,slaves=2,sentinels=3
master23:name=Group_7,status=ok,address=172.30.104.126:60007,slaves=2,sentinels=3
Please see https://gist.github.com/mezzatto/d2a0255f00f97a96d5e0 for the configuration files.
Please see https://gist.github.com/mezzatto/699c38c55354f15cff71. It cointains the ps output taken every 1 second em shows the memory growing.
On 16 Sep the machine 2 failed and the memory started growing. The memory drop today (18 sep, about 11:30 am) happened when I kill the redis-sentinel process:
Even after I restarted the process, the memory leak still happened.
Metadata
Metadata
Assignees
Labels
No labels