Merge pull request #3943 from Algo-devops-service/relbeta3.6.1

Author: algojohnlee

buildnumber.dat

@@ -1 +1 @@
-0
+1

ledger/acctupdates.go

@@ -895,9 +895,10 @@ func (au *accountUpdates) lookupLatest(addr basics.Address) (data basics.Account
 	addResource := func(cidx basics.CreatableIndex, round basics.Round, res ledgercore.AccountResource) error {
 		foundRound, ok := foundResources[cidx]
 		if !ok { // first time seeing this cidx
-			resourceCount++
 			foundResources[cidx] = round
-			ledgercore.AssignAccountResourceToAccountData(cidx, res, &data)
+			if ledgercore.AssignAccountResourceToAccountData(cidx, res, &data) {
+				resourceCount++
+			}
 			return nil
 		}
 		// is this newer than current "found" rnd for this resource?

ledger/acctupdates_test.go

@@ -2357,6 +2357,74 @@ func TestAcctUpdatesLookupRetry(t *testing.T) {
 		})
 }
 
+// auCommitSync is a helper function calling the committing sequence similarly to what tracker registry does
+func auCommitSync(t *testing.T, rnd basics.Round, au *accountUpdates, ml *mockLedgerForTracker) {
+	_, maxLookback := au.committedUpTo(rnd)
+	dcc := &deferredCommitContext{
+		deferredCommitRange: deferredCommitRange{
+			lookback: maxLookback,
+		},
+	}
+	cdr := &dcc.deferredCommitRange
+	cdr = au.produceCommittingTask(rnd, ml.trackers.dbRound, cdr)
+	if cdr != nil {
+		func() {
+			dcc.deferredCommitRange = *cdr
+			ml.trackers.accountsWriting.Add(1)
+			defer ml.trackers.accountsWriting.Done()
+
+			// do not take any locks since all operations are synchronous
+			newBase := basics.Round(dcc.offset) + dcc.oldBase
+			dcc.newBase = newBase
+
+			err := au.prepareCommit(dcc)
+			require.NoError(t, err)
+			err = ml.trackers.dbs.Wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) {
+				err = au.commitRound(ctx, tx, dcc)
+				if err != nil {
+					return err
+				}
+				err = updateAccountsRound(tx, newBase)
+				return err
+			})
+			require.NoError(t, err)
+			ml.trackers.dbRound = newBase
+			au.postCommit(ml.trackers.ctx, dcc)
+			au.postCommitUnlocked(ml.trackers.ctx, dcc)
+		}()
+	}
+}
+
+type auNewBlockOpts struct {
+	updates         ledgercore.AccountDeltas
+	version         protocol.ConsensusVersion
+	protoParams     config.ConsensusParams
+	knownCreatables map[basics.CreatableIndex]bool
+}
+
+func auNewBlock(t *testing.T, rnd basics.Round, au *accountUpdates, base map[basics.Address]basics.AccountData, data auNewBlockOpts) {
+	rewardLevel := uint64(0)
+	prevRound, prevTotals, err := au.LatestTotals()
+	require.Equal(t, rnd-1, prevRound)
+	require.NoError(t, err)
+
+	newTotals := ledgertesting.CalculateNewRoundAccountTotals(t, data.updates, rewardLevel, data.protoParams, base, prevTotals)
+
+	blk := bookkeeping.Block{
+		BlockHeader: bookkeeping.BlockHeader{
+			Round: basics.Round(rnd),
+		},
+	}
+	blk.RewardsLevel = rewardLevel
+	blk.CurrentProtocol = data.version
+	delta := ledgercore.MakeStateDelta(&blk.BlockHeader, 0, data.updates.Len(), 0)
+	delta.Accts.MergeAccounts(data.updates)
+	delta.Creatables = creatablesFromUpdates(base, data.updates, data.knownCreatables)
+	delta.Totals = newTotals
+
+	au.newBlock(blk, delta)
+}
+
 // TestAcctUpdatesLookupLatestCacheRetry simulates a situation when base account and resources are in a cache but
 // account updates advances while calling lookupLatest
 // The idea of the test:
@@ -2408,67 +2476,6 @@ func TestAcctUpdatesLookupLatestCacheRetry(t *testing.T) {
 	aidx2 := basics.AssetIndex(2)
 	knownCreatables := make(map[basics.CreatableIndex]bool)
 
-	commitSync := func(rnd basics.Round) {
-		_, maxLookback := au.committedUpTo(rnd)
-		dcc := &deferredCommitContext{
-			deferredCommitRange: deferredCommitRange{
-				lookback: maxLookback,
-			},
-		}
-		cdr := &dcc.deferredCommitRange
-		cdr = au.produceCommittingTask(rnd, ml.trackers.dbRound, cdr)
-		if cdr != nil {
-			func() {
-				dcc.deferredCommitRange = *cdr
-				ml.trackers.accountsWriting.Add(1)
-				defer ml.trackers.accountsWriting.Done()
-
-				// do not take any locks since all operations are synchronous
-				newBase := basics.Round(dcc.offset) + dcc.oldBase
-				dcc.newBase = newBase
-
-				err := au.prepareCommit(dcc)
-				require.NoError(t, err)
-				err = ml.trackers.dbs.Wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) {
-					err = au.commitRound(ctx, tx, dcc)
-					if err != nil {
-						return err
-					}
-					err = updateAccountsRound(tx, newBase)
-					return err
-				})
-				require.NoError(t, err)
-				ml.trackers.dbRound = newBase
-				au.postCommit(ml.trackers.ctx, dcc)
-				au.postCommitUnlocked(ml.trackers.ctx, dcc)
-			}()
-
-		}
-	}
-
-	newBlock := func(au *accountUpdates, rnd basics.Round, base map[basics.Address]basics.AccountData, updates ledgercore.AccountDeltas) {
-		rewardLevel := uint64(0)
-		prevRound, prevTotals, err := au.LatestTotals()
-		require.Equal(t, rnd-1, prevRound)
-		require.NoError(t, err)
-
-		newTotals := ledgertesting.CalculateNewRoundAccountTotals(t, updates, rewardLevel, protoParams, base, prevTotals)
-
-		blk := bookkeeping.Block{
-			BlockHeader: bookkeeping.BlockHeader{
-				Round: basics.Round(rnd),
-			},
-		}
-		blk.RewardsLevel = rewardLevel
-		blk.CurrentProtocol = testProtocolVersion
-		delta := ledgercore.MakeStateDelta(&blk.BlockHeader, 0, updates.Len(), 0)
-		delta.Accts.MergeAccounts(updates)
-		delta.Creatables = creatablesFromUpdates(base, updates, knownCreatables)
-		delta.Totals = newTotals
-
-		au.newBlock(blk, delta)
-	}
-
 	// the test 1 requires 2 blocks with different resource state, au requires MaxBalLookback block to start persisting
 	for i := basics.Round(1); i <= basics.Round(protoParams.MaxBalLookback+2); i++ {
 		var updates ledgercore.AccountDeltas
@@ -2485,10 +2492,11 @@ func TestAcctUpdatesLookupLatestCacheRetry(t *testing.T) {
 		accts = append(accts, newAccts)
 
 		// prepare block
-		newBlock(au, i, base, updates)
+		opts := auNewBlockOpts{updates, testProtocolVersion, protoParams, knownCreatables}
+		auNewBlock(t, i, au, base, opts)
 
 		// commit changes synchroniously
-		commitSync(i)
+		auCommitSync(t, i, au, ml)
 	}
 
 	// ensure rounds
@@ -2548,8 +2556,9 @@ func TestAcctUpdatesLookupLatestCacheRetry(t *testing.T) {
 	au.accountsMu.Lock()
 	au.cachedDBRound = oldCachedDBRound
 	au.accountsMu.Unlock()
-	newBlock(au, rnd+1, accts[rnd], ledgercore.AccountDeltas{})
-	commitSync(rnd + 1)
+	opts := auNewBlockOpts{ledgercore.AccountDeltas{}, testProtocolVersion, protoParams, knownCreatables}
+	auNewBlock(t, rnd+1, au, accts[rnd], opts)
+	auCommitSync(t, rnd+1, au, ml)
 
 	wg.Wait()
 
@@ -2559,3 +2568,91 @@ func TestAcctUpdatesLookupLatestCacheRetry(t *testing.T) {
 	require.Equal(t, uint64(100), ad.Assets[aidx1].Amount)
 	require.Equal(t, uint64(200), ad.Assets[aidx2].Amount)
 }
+
+// TestAcctUpdatesLookupResources creates 3 assets, deletes one
+// and checks au.resources with deleted resources are not counted toward totals
+func TestAcctUpdatesLookupResources(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
+	accts := []map[basics.Address]basics.AccountData{ledgertesting.RandomAccounts(1, true)}
+	pooldata := basics.AccountData{}
+	pooldata.MicroAlgos.Raw = 100 * 1000 * 1000 * 1000 * 1000
+	pooldata.Status = basics.NotParticipating
+	accts[0][testPoolAddr] = pooldata
+
+	sinkdata := basics.AccountData{}
+	sinkdata.MicroAlgos.Raw = 1000 * 1000 * 1000 * 1000
+	sinkdata.Status = basics.NotParticipating
+	accts[0][testSinkAddr] = sinkdata
+
+	testProtocolVersion := protocol.ConsensusVersion("test-protocol-TestAcctUpdatesLookupResources")
+	protoParams := config.Consensus[protocol.ConsensusCurrentVersion]
+	protoParams.MaxBalLookback = 2
+	protoParams.SeedLookback = 1
+	protoParams.SeedRefreshInterval = 1
+	config.Consensus[testProtocolVersion] = protoParams
+	defer func() {
+		delete(config.Consensus, testProtocolVersion)
+	}()
+
+	ml := makeMockLedgerForTracker(t, true, 1, testProtocolVersion, accts)
+	defer ml.Close()
+
+	conf := config.GetDefaultLocal()
+	au := newAcctUpdates(t, ml, conf, ".")
+	defer au.close()
+
+	var addr1 basics.Address
+	for addr := range accts[0] {
+		if addr != testSinkAddr && addr != testPoolAddr {
+			addr1 = addr
+			break
+		}
+	}
+
+	aidx1 := basics.AssetIndex(1)
+	aidx2 := basics.AssetIndex(2)
+	aidx3 := basics.AssetIndex(3)
+	knownCreatables := make(map[basics.CreatableIndex]bool)
+
+	// test requires 5 blocks: 1 with aidx1, protoParams.MaxBalLookback empty blocks to commit the first one
+	// and 1 block with aidx2 and aidx3, and another one with aidx2 deleted
+	for i := basics.Round(1); i <= basics.Round(protoParams.MaxBalLookback+3); i++ {
+		var updates ledgercore.AccountDeltas
+
+		// add data
+		if i == 1 {
+			updates.Upsert(addr1, ledgercore.AccountData{AccountBaseData: ledgercore.AccountBaseData{MicroAlgos: basics.MicroAlgos{Raw: 1000000}, TotalAssets: 1}})
+			updates.UpsertAssetResource(addr1, aidx1, ledgercore.AssetParamsDelta{}, ledgercore.AssetHoldingDelta{Holding: &basics.AssetHolding{Amount: 100}})
+		}
+		if i == basics.Round(protoParams.MaxBalLookback+2) {
+			updates.Upsert(addr1, ledgercore.AccountData{AccountBaseData: ledgercore.AccountBaseData{MicroAlgos: basics.MicroAlgos{Raw: 1000000}, TotalAssets: 3}})
+			updates.UpsertAssetResource(addr1, aidx2, ledgercore.AssetParamsDelta{}, ledgercore.AssetHoldingDelta{Holding: &basics.AssetHolding{Amount: 200}})
+			updates.UpsertAssetResource(addr1, aidx3, ledgercore.AssetParamsDelta{}, ledgercore.AssetHoldingDelta{Holding: &basics.AssetHolding{Amount: 300}})
+		}
+		if i == basics.Round(protoParams.MaxBalLookback+3) {
+			updates.Upsert(addr1, ledgercore.AccountData{AccountBaseData: ledgercore.AccountBaseData{MicroAlgos: basics.MicroAlgos{Raw: 1000000}, TotalAssets: 2}})
+			updates.UpsertAssetResource(addr1, aidx2, ledgercore.AssetParamsDelta{}, ledgercore.AssetHoldingDelta{Deleted: true})
+		}
+
+		base := accts[i-1]
+		newAccts := applyPartialDeltas(base, updates)
+		accts = append(accts, newAccts)
+
+		// prepare block
+		opts := auNewBlockOpts{updates, testProtocolVersion, protoParams, knownCreatables}
+		auNewBlock(t, i, au, base, opts)
+
+		if i <= basics.Round(protoParams.MaxBalLookback+1) {
+			auCommitSync(t, i, au, ml)
+		}
+		// do not commit two last blocks to keep data in memory deltas
+	}
+	data, rnd, _, err := au.lookupLatest(addr1)
+	require.NoError(t, err)
+	require.Equal(t, basics.Round(protoParams.MaxBalLookback+3), rnd)
+	require.Len(t, data.Assets, 2)
+	require.Contains(t, data.Assets, aidx1)
+	require.Contains(t, data.Assets, aidx3)
+	require.NotContains(t, data.Assets, aidx2)
+}

ledger/ledgercore/accountresource.go

@@ -40,31 +40,38 @@ type AppResource struct {
 	AppParams     *basics.AppParams
 }
 
-// AssignAccountResourceToAccountData assignes the Asset/App params/holdings contained
+// AssignAccountResourceToAccountData assigns the Asset/App params/holdings contained
 // in the AccountResource to the given basics.AccountData, creating maps if necessary.
-func AssignAccountResourceToAccountData(cindex basics.CreatableIndex, resource AccountResource, ad *basics.AccountData) {
+// Returns true if the AccountResource contained a new or updated resource,
+// and false if the AccountResource contained no changes (indicating the resource was deleted).
+func AssignAccountResourceToAccountData(cindex basics.CreatableIndex, resource AccountResource, ad *basics.AccountData) (assigned bool) {
 	if resource.AssetParams != nil {
 		if ad.AssetParams == nil {
 			ad.AssetParams = make(map[basics.AssetIndex]basics.AssetParams)
 		}
 		ad.AssetParams[basics.AssetIndex(cindex)] = *resource.AssetParams
+		assigned = true
 	}
 	if resource.AssetHolding != nil {
 		if ad.Assets == nil {
 			ad.Assets = make(map[basics.AssetIndex]basics.AssetHolding)
 		}
 		ad.Assets[basics.AssetIndex(cindex)] = *resource.AssetHolding
+		assigned = true
 	}
 	if resource.AppParams != nil {
 		if ad.AppParams == nil {
 			ad.AppParams = make(map[basics.AppIndex]basics.AppParams)
 		}
 		ad.AppParams[basics.AppIndex(cindex)] = *resource.AppParams
+		assigned = true
 	}
 	if resource.AppLocalState != nil {
 		if ad.AppLocalStates == nil {
 			ad.AppLocalStates = make(map[basics.AppIndex]basics.AppLocalState)
 		}
 		ad.AppLocalStates[basics.AppIndex(cindex)] = *resource.AppLocalState
+		assigned = true
 	}
+	return
 }