core,eth,les: update unclean shutdown markers regularly

core/rawdb/accessors_metadata.go

@@ -139,6 +139,28 @@ func PopUncleanShutdownMarker(db ethdb.KeyValueStore) {
 	}
 }
 
+// UpdateUncleanShutdownMarker updates the last marker's timestamp to now.
+func UpdateUncleanShutdownMarker(db ethdb.KeyValueStore) {
+	var uncleanShutdowns crashList
+	// Read old data
+	if data, err := db.Get(uncleanShutdownKey); err != nil {
+		log.Warn("Error reading unclean shutdown markers", "error", err)
+	} else if err := rlp.DecodeBytes(data, &uncleanShutdowns); err != nil {
+		log.Warn("Error decoding unclean shutdown markers", "error", err)
+	}
+	// This shouldn't happen because we push a marker on Backend instantiation
+	count := len(uncleanShutdowns.Recent)
+	if count == 0 {
+		log.Warn("No unclean shutdown marker to update")
+		return
+	}
+	uncleanShutdowns.Recent[count-1] = uint64(time.Now().Unix())
+	data, _ := rlp.EncodeToBytes(uncleanShutdowns)
+	if err := db.Put(uncleanShutdownKey, data); err != nil {
+		log.Warn("Failed to write unclean-shutdown marker", "err", err)
+	}
+}
+
 // ReadTransitionStatus retrieves the eth2 transition status from the database
 func ReadTransitionStatus(db ethdb.KeyValueReader) []byte {
 	data, _ := db.Get(transitionStatusKey)

eth/backend.go

@@ -97,6 +97,8 @@ type Ethereum struct {
 	p2pServer *p2p.Server
 
 	lock sync.RWMutex // Protects the variadic fields (e.g. gas price and etherbase)
+
+	shutdownCh chan chan struct{} // Signals the unclean shutdown marker updating loop to terminate
 }
 
 // New creates a new Ethereum object (including the
@@ -157,6 +159,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 		bloomRequests:     make(chan chan *bloombits.Retrieval),
 		bloomIndexer:      core.NewBloomIndexer(chainDb, params.BloomBitsBlocks, params.BloomConfirms),
 		p2pServer:         stack.Server(),
+		shutdownCh:        make(chan chan struct{}),
 	}
 
 	bcVersion := rawdb.ReadDatabaseVersion(chainDb)
@@ -275,6 +278,9 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 				"age", common.PrettyAge(t))
 		}
 	}
+	// Keep updating shutdown markers to pin down on the actual
+	// time of the crash.
+	go eth.updateUncleanShutdownMarkers()
 	return eth, nil
 }
 
@@ -577,9 +583,31 @@ func (s *Ethereum) Stop() error {
 	s.miner.Close()
 	s.blockchain.Stop()
 	s.engine.Close()
-	rawdb.PopUncleanShutdownMarker(s.chainDb)
+	// Stop updating unclean shutdown markers and pop the last marker.
+	done := make(chan struct{})
+	s.shutdownCh <- done
+	<-done
+
 	s.chainDb.Close()
 	s.eventMux.Stop()
 
 	return nil
 }
+
+func (s *Ethereum) updateUncleanShutdownMarkers() {
+	// update marker every five minutes
+	ticker := time.NewTicker(300 * time.Second)
+	defer func() { ticker.Stop() }()
+	for {
+		select {
+		case <-ticker.C:
+			rawdb.UpdateUncleanShutdownMarker(s.chainDb)
+		case done := <-s.shutdownCh:
+			// Successful shutdown, clear last marker
+			rawdb.PopUncleanShutdownMarker(s.chainDb)
+			done <- struct{}{}
+			// ticker stops upon return
+			return
+		}
+	}
+}

les/client.go

@@ -77,6 +77,8 @@ type LightEthereum struct {
 	p2pServer  *p2p.Server
 	p2pConfig  *p2p.Config
 	udpEnabled bool
+
+	shutdownCh chan chan struct{} // Signals the unclean shutdown marker updating loop to terminate
 }
 
 // New creates an instance of the light client.
@@ -118,6 +120,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) {
 		p2pServer:      stack.Server(),
 		p2pConfig:      &stack.Config().P2P,
 		udpEnabled:     stack.Config().P2P.DiscoveryV5,
+		shutdownCh:     make(chan chan struct{}),
 	}
 
 	var prenegQuery vfc.QueryFunc
@@ -198,6 +201,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) {
 				"age", common.PrettyAge(t))
 		}
 	}
+	go leth.updateUncleanShutdownMarkers()
 	return leth, nil
 }
 
@@ -387,10 +391,32 @@ func (s *LightEthereum) Stop() error {
 	s.engine.Close()
 	s.pruner.close()
 	s.eventMux.Stop()
-	rawdb.PopUncleanShutdownMarker(s.chainDb)
+	// Stop updating unclean shutdown markers and pop the last marker.
+	done := make(chan struct{})
+	s.shutdownCh <- done
+	<-done
+
 	s.chainDb.Close()
 	s.lesDb.Close()
 	s.wg.Wait()
 	log.Info("Light ethereum stopped")
 	return nil
 }
+
+func (s *LightEthereum) updateUncleanShutdownMarkers() {
+	// update marker every five minutes
+	ticker := time.NewTicker(300 * time.Second)
+	defer func() { ticker.Stop() }()
+	for {
+		select {
+		case <-ticker.C:
+			rawdb.UpdateUncleanShutdownMarker(s.chainDb)
+		case done := <-s.shutdownCh:
+			// Successful shutdown, clear last marker
+			rawdb.PopUncleanShutdownMarker(s.chainDb)
+			done <- struct{}{}
+			// ticker stops upon return
+			return
+		}
+	}
+}