From fdd16f211ede4413fd66b339513f0f64452240e4 Mon Sep 17 00:00:00 2001 From: Jay Date: Mon, 13 Feb 2017 13:25:24 +0800 Subject: [PATCH] raftstore: use static lease (#1594) * raftstore: use static lease --- src/raftstore/store/config.rs | 20 ++++++++++++++++++-- src/raftstore/store/peer.rs | 15 +++++++-------- src/raftstore/store/store.rs | 10 +++++----- tests/raftstore/test_lease_read.rs | 9 +++++++-- tests/raftstore/util.rs | 2 ++ 5 files changed, 39 insertions(+), 17 deletions(-) diff --git a/src/raftstore/store/config.rs b/src/raftstore/store/config.rs index f5565bf1256..9960a081a8c 100644 --- a/src/raftstore/store/config.rs +++ b/src/raftstore/store/config.rs @@ -14,11 +14,13 @@ use std::u64; use std::time::Duration; +use time::Duration as TimeDuration; + use raftstore::Result; const RAFT_BASE_TICK_INTERVAL: u64 = 100; -const RAFT_HEARTBEAT_TICKS: usize = 10; -const RAFT_ELECTION_TIMEOUT_TICKS: usize = 50; +const RAFT_HEARTBEAT_TICKS: usize = 20; +const RAFT_ELECTION_TIMEOUT_TICKS: usize = 100; const RAFT_MAX_SIZE_PER_MSG: u64 = 1024 * 1024; const RAFT_MAX_INFLIGHT_MSGS: usize = 256; const RAFT_ENTRY_MAX_SIZE: u64 = 8 * 1024 * 1024; @@ -52,6 +54,8 @@ const DEFAULT_CONSISTENCY_CHECK_INTERVAL: u64 = 0; const DEFAULT_REPORT_REGION_FLOW_INTERVAL: u64 = 30000; // 30 seconds +const DEFAULT_RAFT_STORE_LEASE_SEC: i64 = 9; // 9 seconds + #[derive(Debug, Clone)] pub struct Config { // store capacity. @@ -117,6 +121,8 @@ pub struct Config { pub consistency_check_tick_interval: u64, pub report_region_flow_interval: u64, + // The lease provided by a successfully proposed and applied entry. + pub raft_store_max_leader_lease: TimeDuration, } impl Default for Config { @@ -151,6 +157,7 @@ impl Default for Config { lock_cf_compact_interval: DEFAULT_LOCK_CF_COMPACT_INTERVAL, consistency_check_tick_interval: DEFAULT_CONSISTENCY_CHECK_INTERVAL, report_region_flow_interval: DEFAULT_REPORT_REGION_FLOW_INTERVAL, + raft_store_max_leader_lease: TimeDuration::seconds(DEFAULT_RAFT_STORE_LEASE_SEC), } } } @@ -192,6 +199,15 @@ impl Config { self.region_split_size)); } + let election_timeout = self.raft_base_tick_interval * + self.raft_election_timeout_ticks as u64; + let lease = self.raft_store_max_leader_lease.num_milliseconds() as u64; + if election_timeout < lease { + return Err(box_err!("election timeout {} ms is less than lease {} ms", + election_timeout, + lease)); + } + Ok(()) } } diff --git a/src/raftstore/store/peer.rs b/src/raftstore/store/peer.rs index 77d58359d9f..f15cd17c773 100644 --- a/src/raftstore/store/peer.rs +++ b/src/raftstore/store/peer.rs @@ -18,7 +18,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::vec::Vec; use std::default::Default; use std::time::{Instant, Duration}; -use time::{Timespec, Duration as TimeDuration}; +use time::Timespec; use rocksdb::{DB, WriteBatch, Writable}; use protobuf::{self, Message, MessageStatic}; @@ -244,6 +244,7 @@ pub struct ConsistencyState { pub struct Peer { engine: Arc, + cfg: Rc, peer_cache: Rc>>, pub peer: metapb::Peer, region_id: u64, @@ -275,7 +276,6 @@ pub struct Peer { leader_missing_time: Option, leader_lease_expired_time: Option, - election_timeout: TimeDuration, pub written_bytes: u64, pub written_keys: u64, @@ -365,9 +365,8 @@ impl Peer { }, raft_log_size_hint: 0, raft_entry_max_size: cfg.raft_entry_max_size, + cfg: cfg, leader_lease_expired_time: None, - election_timeout: TimeDuration::milliseconds(cfg.raft_base_tick_interval as i64) * - cfg.raft_election_timeout_ticks as i32, written_bytes: 0, written_keys: 0, }; @@ -591,10 +590,10 @@ impl Peer { fn next_lease_expired_time(&self, send_to_quorum_ts: Timespec) -> Timespec { // The valid leader lease should be - // "lease = election_timeout - (quorum_commit_ts - send_to_quorum_ts)" + // "lease = max_lease - (quorum_commit_ts - send_to_quorum_ts)" // And the expired timestamp for that leader lease is "quorum_commit_ts + lease", - // which is "send_to_quorum_ts + election_timeout" in short. - send_to_quorum_ts + self.election_timeout + // which is "send_to_quorum_ts + max_lease" in short. + send_to_quorum_ts + self.cfg.raft_store_max_leader_lease } fn update_leader_lease(&mut self, ready: &Ready) { @@ -606,7 +605,7 @@ impl Peer { // The local read can only be performed after a new leader has applied // the first empty entry on its term. After that the lease expiring time // should be updated to - // send_to_quorum_ts + election_timeout + // send_to_quorum_ts + max_lease // as the comments in `next_lease_expired_time` function explain. // It is recommended to update the lease expiring time right after // this peer becomes leader because it's more convenient to do it here and diff --git a/src/raftstore/store/store.rs b/src/raftstore/store/store.rs index c4d2dc45a53..4442b1003e2 100644 --- a/src/raftstore/store/store.rs +++ b/src/raftstore/store/store.rs @@ -76,9 +76,9 @@ pub struct StoreChannel { } pub struct Store { - cfg: Config, - store: metapb::Store, + cfg: Rc, engine: Arc, + store: metapb::Store, sendch: SendCh, sent_snapshot_count: u64, @@ -158,7 +158,7 @@ impl Store { let tag = format!("[store {}]", meta.get_id()); let mut s = Store { - cfg: cfg, + cfg: Rc::new(cfg), store: meta, engine: engine, sendch: sendch, @@ -294,8 +294,8 @@ impl Store { self.store.get_id() } - pub fn config(&self) -> &Config { - &self.cfg + pub fn config(&self) -> Rc { + self.cfg.clone() } pub fn peer_cache(&self) -> Rc>> { diff --git a/tests/raftstore/test_lease_read.rs b/tests/raftstore/test_lease_read.rs index 1dddc60f33e..1b5ac069126 100644 --- a/tests/raftstore/test_lease_read.rs +++ b/tests/raftstore/test_lease_read.rs @@ -16,6 +16,8 @@ use std::thread; use std::time::Duration; +use time::Duration as TimeDuration; + use kvproto::metapb::{Peer, Region}; use kvproto::raft_cmdpb::CmdType; use kvproto::raft_serverpb::RaftLocalState; @@ -94,8 +96,11 @@ fn test_renew_lease(cluster: &mut Cluster) { cluster.cfg.raft_store.raft_base_tick_interval = 50; cluster.cfg.raft_store.raft_election_timeout_ticks = 90; - let election_timeout = Duration::from_millis(cluster.cfg.raft_store.raft_base_tick_interval) * - cluster.cfg.raft_store.raft_election_timeout_ticks as u32; + let election_timeout = + Duration::from_millis(cluster.cfg.raft_store.raft_base_tick_interval * + cluster.cfg.raft_store.raft_election_timeout_ticks as u64); + cluster.cfg.raft_store.raft_store_max_leader_lease = TimeDuration::from_std(election_timeout) + .unwrap(); let node_id = 1u64; let store_id = 1u64; diff --git a/tests/raftstore/util.rs b/tests/raftstore/util.rs index 7436a3475e4..3744e3a4e52 100644 --- a/tests/raftstore/util.rs +++ b/tests/raftstore/util.rs @@ -19,6 +19,7 @@ use std::thread; use rocksdb::DB; use uuid::Uuid; use protobuf; +use time::Duration as TimeDuration; use kvproto::metapb::{self, RegionEpoch}; use kvproto::raft_cmdpb::{Request, StatusRequest, AdminRequest, RaftCmdRequest, RaftCmdResponse}; @@ -87,6 +88,7 @@ pub fn new_store_cfg() -> Config { // should be configured far beyond the election timeout. max_leader_missing_duration: Duration::from_secs(3), report_region_flow_interval: 100, // 100ms + raft_store_max_leader_lease: TimeDuration::milliseconds(25 * 10), ..Config::default() } }