Merge pull request #111 from BoostryJP/109-feature

Optimize trie database

Author: YoshihitoAso

eth/fetcher/block_fetcher.go

@@ -832,15 +832,17 @@ func (f *BlockFetcher) importBlocks(peer string, block *types.Block) {
 // internal state.
 func (f *BlockFetcher) forgetHash(hash common.Hash) {
 	// Remove all pending announces and decrement DOS counters
-	for _, announce := range f.announced[hash] {
-		f.announces[announce.origin]--
-		if f.announces[announce.origin] <= 0 {
-			delete(f.announces, announce.origin)
+	if announceMap, ok := f.announced[hash]; ok {
+		for _, announce := range announceMap {
+			f.announces[announce.origin]--
+			if f.announces[announce.origin] <= 0 {
+				delete(f.announces, announce.origin)
+			}
+		}
+		delete(f.announced, hash)
+		if f.announceChangeHook != nil {
+			f.announceChangeHook(hash, false)
 		}
-	}
-	delete(f.announced, hash)
-	if f.announceChangeHook != nil {
-		f.announceChangeHook(hash, false)
 	}
 	// Remove any pending fetches and decrement the DOS counters
 	if announce := f.fetching[hash]; announce != nil {

eth/fetcher/block_fetcher_test.go

@@ -698,6 +698,7 @@ func testInvalidNumberAnnouncement(t *testing.T, light bool) {
 	badBodyFetcher := tester.makeBodyFetcher("bad", blocks, 0)
 
 	imported := make(chan interface{})
+	announced := make(chan interface{})
 	tester.fetcher.importedHook = func(header *types.Header, block *types.Block) {
 		if light {
 			if header == nil {
@@ -712,21 +713,35 @@ func testInvalidNumberAnnouncement(t *testing.T, light bool) {
 		}
 	}
 	// Announce a block with a bad number, check for immediate drop
+	tester.fetcher.announceChangeHook = func(hash common.Hash, b bool) {
+		announced <- nil
+	}
 	tester.fetcher.Notify("bad", hashes[0], 2, time.Now().Add(-arriveTimeout), badHeaderFetcher, badBodyFetcher)
+	verifyAnnounce := func() {
+		for i := 0; i < 2; i++ {
+			select {
+			case <-announced:
+				continue
+			case <-time.After(1 * time.Second):
+				t.Fatal("announce timeout")
+				return
+			}
+		}
+	}
+	verifyAnnounce()
 	verifyImportEvent(t, imported, false)
-
 	tester.lock.RLock()
 	dropped := tester.drops["bad"]
 	tester.lock.RUnlock()
 
 	if !dropped {
 		t.Fatalf("peer with invalid numbered announcement not dropped")
 	}
-
 	goodHeaderFetcher := tester.makeHeaderFetcher("good", blocks, -gatherSlack)
 	goodBodyFetcher := tester.makeBodyFetcher("good", blocks, 0)
 	// Make sure a good announcement passes without a drop
 	tester.fetcher.Notify("good", hashes[0], 1, time.Now().Add(-arriveTimeout), goodHeaderFetcher, goodBodyFetcher)
+	verifyAnnounce()
 	verifyImportEvent(t, imported, true)
 
 	tester.lock.RLock()

les/pruner.go

@@ -62,6 +62,7 @@ func (p *pruner) loop() {
 
 	// cleanTicker is the ticker used to trigger a history clean 2 times a day.
 	var cleanTicker = time.NewTicker(12 * time.Hour)
+	defer cleanTicker.Stop()
 
 	// pruning finds the sections that have been processed by all indexers
 	// and deletes all historical chain data.

trie/database.go

@@ -703,12 +703,6 @@ func (db *Database) Commit(node common.Hash, report bool, callback func(common.H
 	// Move all of the accumulated preimages into a write batch
 	if db.preimages != nil {
 		rawdb.WritePreimages(batch, db.preimages)
-		if batch.ValueSize() > ethdb.IdealBatchSize {
-			if err := batch.Write(); err != nil {
-				return err
-			}
-			batch.Reset()
-		}
 		// Since we're going to replay trie node writes into the clean cache, flush out
 		// any batched pre-images before continuing.
 		if err := batch.Write(); err != nil {

trie/sync_bloom.go

@@ -45,11 +45,12 @@ var (
 // provided disk database on creation in a background thread and will only start
 // returning live results once that's finished.
 type SyncBloom struct {
-	bloom  *bloomfilter.Filter
-	inited uint32
-	closer sync.Once
-	closed uint32
-	pend   sync.WaitGroup
+	bloom   *bloomfilter.Filter
+	inited  uint32
+	closer  sync.Once
+	closed  uint32
+	pend    sync.WaitGroup
+	closeCh chan struct{}
 }
 
 // NewSyncBloom creates a new bloom filter of the given size (in megabytes) and
@@ -64,7 +65,8 @@ func NewSyncBloom(memory uint64, database ethdb.Iteratee) *SyncBloom {
 
 	// Assemble the fast sync bloom and init it from previous sessions
 	b := &SyncBloom{
-		bloom: bloom,
+		bloom:   bloom,
+		closeCh: make(chan struct{}),
 	}
 	b.pend.Add(2)
 	go func() {
@@ -125,16 +127,17 @@ func (b *SyncBloom) init(database ethdb.Iteratee) {
 // meter periodically recalculates the false positive error rate of the bloom
 // filter and reports it in a metric.
 func (b *SyncBloom) meter() {
+	// check every second
+	tick := time.NewTicker(1 * time.Second)
+	defer tick.Stop()
+
 	for {
-		// Report the current error ration. No floats, lame, scale it up.
-		bloomErrorGauge.Update(int64(b.bloom.FalsePosititveProbability() * 100000))
-
-		// Wait one second, but check termination more frequently
-		for i := 0; i < 10; i++ {
-			if atomic.LoadUint32(&b.closed) == 1 {
-				return
-			}
-			time.Sleep(100 * time.Millisecond)
+		select {
+		case <-tick.C:
+			// Report the current error ration. No floats, lame, scale it up.
+			bloomErrorGauge.Update(int64(b.bloom.FalsePosititveProbability() * 100000))
+		case <-b.closeCh:
+			return
 		}
 	}
 }
@@ -145,6 +148,7 @@ func (b *SyncBloom) Close() error {
 	b.closer.Do(func() {
 		// Ensure the initializer is stopped
 		atomic.StoreUint32(&b.closed, 1)
+		close(b.closeCh)
 		b.pend.Wait()
 
 		// Wipe the bloom, but mark it "uninited" just in case someone attempts an access

trie/trie.go

@@ -405,6 +405,14 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) {
 		n.flags = t.newFlag()
 		n.Children[key[0]] = nn
 
+		// Because n is a full node, it must've contained at least two children
+		// before the delete operation. If the new child value is non-nil, n still
+		// has at least two children after the deletion, and cannot be reduced to
+		// a short node.
+		if nn != nil {
+			return true, n, nil
+		}
+		// Reduction:
 		// Check how many non-nil entries are left after deleting and
 		// reduce the full node to a short node if only one entry is
 		// left. Since n must've contained at least two children