raftstore: use static lease (tikv#1594)

* raftstore: use static lease

src/raftstore/store/config.rs

@@ -14,11 +14,13 @@
 use std::u64;
 use std::time::Duration;
 
+use time::Duration as TimeDuration;
+
 use raftstore::Result;
 
 const RAFT_BASE_TICK_INTERVAL: u64 = 100;
-const RAFT_HEARTBEAT_TICKS: usize = 10;
-const RAFT_ELECTION_TIMEOUT_TICKS: usize = 50;
+const RAFT_HEARTBEAT_TICKS: usize = 20;
+const RAFT_ELECTION_TIMEOUT_TICKS: usize = 100;
 const RAFT_MAX_SIZE_PER_MSG: u64 = 1024 * 1024;
 const RAFT_MAX_INFLIGHT_MSGS: usize = 256;
 const RAFT_ENTRY_MAX_SIZE: u64 = 8 * 1024 * 1024;
@@ -52,6 +54,8 @@ const DEFAULT_CONSISTENCY_CHECK_INTERVAL: u64 = 0;
 
 const DEFAULT_REPORT_REGION_FLOW_INTERVAL: u64 = 30000; // 30 seconds
 
+const DEFAULT_RAFT_STORE_LEASE_SEC: i64 = 9; // 9 seconds
+
 #[derive(Debug, Clone)]
 pub struct Config {
     // store capacity.
@@ -117,6 +121,8 @@ pub struct Config {
     pub consistency_check_tick_interval: u64,
 
     pub report_region_flow_interval: u64,
+    // The lease provided by a successfully proposed and applied entry.
+    pub raft_store_max_leader_lease: TimeDuration,
 }
 
 impl Default for Config {
@@ -151,6 +157,7 @@ impl Default for Config {
             lock_cf_compact_interval: DEFAULT_LOCK_CF_COMPACT_INTERVAL,
             consistency_check_tick_interval: DEFAULT_CONSISTENCY_CHECK_INTERVAL,
             report_region_flow_interval: DEFAULT_REPORT_REGION_FLOW_INTERVAL,
+            raft_store_max_leader_lease: TimeDuration::seconds(DEFAULT_RAFT_STORE_LEASE_SEC),
         }
     }
 }
@@ -192,6 +199,15 @@ impl Config {
                                 self.region_split_size));
         }
 
+        let election_timeout = self.raft_base_tick_interval *
+                               self.raft_election_timeout_ticks as u64;
+        let lease = self.raft_store_max_leader_lease.num_milliseconds() as u64;
+        if election_timeout < lease {
+            return Err(box_err!("election timeout {} ms is less than lease {} ms",
+                                election_timeout,
+                                lease));
+        }
+
         Ok(())
     }
 }

src/raftstore/store/peer.rs

@@ -18,7 +18,7 @@ use std::collections::{HashMap, HashSet, VecDeque};
 use std::vec::Vec;
 use std::default::Default;
 use std::time::{Instant, Duration};
-use time::{Timespec, Duration as TimeDuration};
+use time::Timespec;
 
 use rocksdb::{DB, WriteBatch, Writable};
 use protobuf::{self, Message, MessageStatic};
@@ -244,6 +244,7 @@ pub struct ConsistencyState {
 
 pub struct Peer {
     engine: Arc<DB>,
+    cfg: Rc<Config>,
     peer_cache: Rc<RefCell<HashMap<u64, metapb::Peer>>>,
     pub peer: metapb::Peer,
     region_id: u64,
@@ -275,7 +276,6 @@ pub struct Peer {
     leader_missing_time: Option<Instant>,
 
     leader_lease_expired_time: Option<Timespec>,
-    election_timeout: TimeDuration,
 
     pub written_bytes: u64,
     pub written_keys: u64,
@@ -365,9 +365,8 @@ impl Peer {
             },
             raft_log_size_hint: 0,
             raft_entry_max_size: cfg.raft_entry_max_size,
+            cfg: cfg,
             leader_lease_expired_time: None,
-            election_timeout: TimeDuration::milliseconds(cfg.raft_base_tick_interval as i64) *
-                              cfg.raft_election_timeout_ticks as i32,
             written_bytes: 0,
             written_keys: 0,
         };
@@ -591,10 +590,10 @@ impl Peer {
 
     fn next_lease_expired_time(&self, send_to_quorum_ts: Timespec) -> Timespec {
         // The valid leader lease should be
-        // "lease = election_timeout - (quorum_commit_ts - send_to_quorum_ts)"
+        // "lease = max_lease - (quorum_commit_ts - send_to_quorum_ts)"
         // And the expired timestamp for that leader lease is "quorum_commit_ts + lease",
-        // which is "send_to_quorum_ts + election_timeout" in short.
-        send_to_quorum_ts + self.election_timeout
+        // which is "send_to_quorum_ts + max_lease" in short.
+        send_to_quorum_ts + self.cfg.raft_store_max_leader_lease
     }
 
     fn update_leader_lease(&mut self, ready: &Ready) {
@@ -606,7 +605,7 @@ impl Peer {
                     // The local read can only be performed after a new leader has applied
                     // the first empty entry on its term. After that the lease expiring time
                     // should be updated to
-                    //   send_to_quorum_ts + election_timeout
+                    //   send_to_quorum_ts + max_lease
                     // as the comments in `next_lease_expired_time` function explain.
                     // It is recommended to update the lease expiring time right after
                     // this peer becomes leader because it's more convenient to do it here and

src/raftstore/store/store.rs

@@ -76,9 +76,9 @@ pub struct StoreChannel {
 }
 
 pub struct Store<T, C: 'static> {
-    cfg: Config,
-    store: metapb::Store,
+    cfg: Rc<Config>,
     engine: Arc<DB>,
+    store: metapb::Store,
     sendch: SendCh<Msg>,
 
     sent_snapshot_count: u64,
@@ -158,7 +158,7 @@ impl<T, C> Store<T, C> {
         let tag = format!("[store {}]", meta.get_id());
 
         let mut s = Store {
-            cfg: cfg,
+            cfg: Rc::new(cfg),
             store: meta,
             engine: engine,
             sendch: sendch,
@@ -294,8 +294,8 @@ impl<T, C> Store<T, C> {
         self.store.get_id()
     }
 
-    pub fn config(&self) -> &Config {
-        &self.cfg
+    pub fn config(&self) -> Rc<Config> {
+        self.cfg.clone()
     }
 
     pub fn peer_cache(&self) -> Rc<RefCell<HashMap<u64, metapb::Peer>>> {

tests/raftstore/test_lease_read.rs

@@ -16,6 +16,8 @@
 use std::thread;
 use std::time::Duration;
 
+use time::Duration as TimeDuration;
+
 use kvproto::metapb::{Peer, Region};
 use kvproto::raft_cmdpb::CmdType;
 use kvproto::raft_serverpb::RaftLocalState;
@@ -94,8 +96,11 @@ fn test_renew_lease<T: Simulator>(cluster: &mut Cluster<T>) {
     cluster.cfg.raft_store.raft_base_tick_interval = 50;
     cluster.cfg.raft_store.raft_election_timeout_ticks = 90;
 
-    let election_timeout = Duration::from_millis(cluster.cfg.raft_store.raft_base_tick_interval) *
-                           cluster.cfg.raft_store.raft_election_timeout_ticks as u32;
+    let election_timeout =
+        Duration::from_millis(cluster.cfg.raft_store.raft_base_tick_interval *
+                              cluster.cfg.raft_store.raft_election_timeout_ticks as u64);
+    cluster.cfg.raft_store.raft_store_max_leader_lease = TimeDuration::from_std(election_timeout)
+        .unwrap();
 
     let node_id = 1u64;
     let store_id = 1u64;

tests/raftstore/util.rs

@@ -19,6 +19,7 @@ use std::thread;
 use rocksdb::DB;
 use uuid::Uuid;
 use protobuf;
+use time::Duration as TimeDuration;
 
 use kvproto::metapb::{self, RegionEpoch};
 use kvproto::raft_cmdpb::{Request, StatusRequest, AdminRequest, RaftCmdRequest, RaftCmdResponse};
@@ -87,6 +88,7 @@ pub fn new_store_cfg() -> Config {
         // should be configured far beyond the election timeout.
         max_leader_missing_duration: Duration::from_secs(3),
         report_region_flow_interval: 100, // 100ms
+        raft_store_max_leader_lease: TimeDuration::milliseconds(25 * 10),
         ..Config::default()
     }
 }