Skip to content

test: large generated accountdb cluster test #4772

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
algorandskiy merged 17 commits into from
Nov 17, 2022
Merged

test: large generated accountdb cluster test #4772

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
d2d7dc6
wip: s1s bigdb
brianolson Oct 26, 2022
ecd4a71
add --cpuprofile
brianolson Oct 27, 2022
6d812cf
debug logging
brianolson Oct 27, 2022
b18bfb1
fix accounts? debug log
brianolson Oct 27, 2022
376e83e
fix tx generation
brianolson Oct 27, 2022
5b6e279
fix genesis account vote clobbering
brianolson Oct 28, 2022
039b414
do what we need in 400 rounds
brianolson Nov 2, 2022
ce43d60
Merge remote-tracking branch 'origin/master' into s1bigdb
brianolson Nov 2, 2022
c6846a0
make bootstrappedScenario be scenario1s but with db
brianolson Nov 2, 2022
60c80b4
we only need a bit over 1500 rounds to make accounts
brianolson Nov 3, 2022
9bcbc0e
test improvements
brianolson Nov 7, 2022
88aeab2
Merge remote-tracking branch 'origin/master' into s1bigdb
brianolson Nov 7, 2022
1b29f48
lint
brianolson Nov 9, 2022
f9221d1
Merge remote-tracking branch 'origin/master' into s1bigdb
brianolson Nov 10, 2022
991d3e5
Merge remote-tracking branch 'm2/s1bigdb' into s1bigdb
brianolson Nov 10, 2022
229fc6d
fix test
brianolson Nov 14, 2022
b7930a5
Merge remote-tracking branch 'origin/master' into s1bigdb
brianolson Nov 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions cmd/netgoal/network.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"fmt"
"os"
"path/filepath"
"runtime/pprof"
"strings"

"github.com/spf13/cobra"
Expand All @@ -35,6 +36,8 @@ var networkName string
var networkGenesisVersionModifier string
var miscStringStringTokens []string

var cpuprofilePath string

var networkUseGenesisFiles bool
var networkIgnoreExistingDir bool
var bootstrapLoadingFile bool
Expand All @@ -54,6 +57,7 @@ func init() {
networkBuildCmd.Flags().BoolVarP(&bootstrapLoadingFile, "gen-db-files", "b", false, "Generate database files.")
networkBuildCmd.Flags().BoolVarP(&networkIgnoreExistingDir, "force", "f", false, "Force generation into existing directory.")
networkBuildCmd.Flags().StringSliceVarP(&miscStringStringTokens, "val", "v", nil, "name=value, may be reapeated")
networkBuildCmd.Flags().StringVar(&cpuprofilePath, "cpuprofile", "", "write cpu profile to path")

rootCmd.PersistentFlags().StringVarP(&networkGenesisVersionModifier, "modifier", "m", "", "Override Genesis Version Modifier (eg 'v1')")
}
Expand All @@ -75,6 +79,18 @@ var networkBuildCmd = &cobra.Command{
}

func runBuildNetwork() (err error) {
if cpuprofilePath != "" {
f, err := os.Create(cpuprofilePath)
if err != nil {
log.Fatalf("%s: could not create CPU profile, %v", cpuprofilePath, err)
}
defer f.Close() // error handling omitted for example
if err := pprof.StartCPUProfile(f); err != nil {
log.Fatalf("%s: could not start CPU profile, %v", cpuprofilePath, err)
}
defer pprof.StopCPUProfile()
}

networkRootDir, err := filepath.Abs(networkRootDir)
if err != nil {
return
Expand Down
43 changes: 29 additions & 14 deletions netdeploy/remote/deployedNetwork.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package remote

import (
"encoding/binary"
"encoding/json"
"fmt"
"io/fs"
Expand Down Expand Up @@ -103,6 +104,8 @@ type netState struct {
accounts []basics.Address
txnCount uint64
fundPerAccount basics.MicroAlgos

log logging.Logger
}

const program = `#pragma version 2
Expand Down Expand Up @@ -382,10 +385,10 @@ func (cfg DeployedNetwork) GenerateDatabaseFiles(fileCfgs BootstrappedNetwork, g
}

accounts[addr] = alloc.State

}

//initial state
log := logging.NewLogger()
Copy link
Contributor

@algorandskiy algorandskiy Nov 11, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this inits local var, not global one. Maybe move it into init() ?


bootstrappedNet := netState{
nAssets: fileCfgs.GeneratedAssetsCount,
Expand All @@ -397,6 +400,7 @@ func (cfg DeployedNetwork) GenerateDatabaseFiles(fileCfgs BootstrappedNetwork, g
genesisHash: genesis.Hash(),
poolAddr: poolAddr,
sinkAddr: sinkAddr,
log: log,
}

var params config.ConsensusParams
Expand All @@ -420,8 +424,9 @@ func (cfg DeployedNetwork) GenerateDatabaseFiles(fileCfgs BootstrappedNetwork, g
max := fileCfgs.BalanceRange[1]
bal := rand.Int63n(max-min) + min
bootstrappedNet.fundPerAccount = basics.MicroAlgos{Raw: uint64(bal)}
totalFunds := accounts[src].MicroAlgos.Raw + bootstrappedNet.fundPerAccount.Raw*bootstrappedNet.nAccounts + bootstrappedNet.roundTxnCnt*fileCfgs.NumRounds
accounts[src] = basics.MakeAccountData(basics.Online, basics.MicroAlgos{Raw: totalFunds})
srcAcct := accounts[src]
srcAcct.MicroAlgos.Raw += bootstrappedNet.fundPerAccount.Raw*bootstrappedNet.nAccounts + bootstrappedNet.roundTxnCnt*fileCfgs.NumRounds
accounts[src] = srcAcct

//init block
initState, err := generateInitState(accounts, &bootstrappedNet)
Expand All @@ -432,24 +437,24 @@ func (cfg DeployedNetwork) GenerateDatabaseFiles(fileCfgs BootstrappedNetwork, g
localCfg.Archival = true
localCfg.CatchpointTracking = -1
localCfg.LedgerSynchronousMode = 0
log := logging.NewLogger()
l, err := ledger.OpenLedger(log, filepath.Join(genesisFolder, "bootstrapped"), false, initState, localCfg)
if err != nil {
return err
}

//create accounts, apps and assets
prev, _ := l.Block(l.Latest())
err = generateAccounts(src, fileCfgs.RoundTransactionsCount, prev, l, &bootstrappedNet, params)
err = generateAccounts(src, fileCfgs.RoundTransactionsCount, prev, l, &bootstrappedNet, params, log)
if err != nil {
return err
}

log.Info("setup done, more txns")
//create more transactions
prev, _ = l.Block(l.Latest())
for i := uint64(bootstrappedNet.round); i < fileCfgs.NumRounds; i++ {
bootstrappedNet.round++
blk, _ := createBlock(src, prev, fileCfgs.RoundTransactionsCount, &bootstrappedNet, params)
blk, _ := createBlock(src, prev, fileCfgs.RoundTransactionsCount, &bootstrappedNet, params, log)
err = l.AddBlock(blk, agreement.Certificate{Round: bootstrappedNet.round})
if err != nil {
fmt.Printf("Error %v\n", err)
Expand Down Expand Up @@ -516,7 +521,7 @@ func generateInitState(accounts map[basics.Address]basics.AccountData, bootstrap
return initState, nil
}

func createBlock(src basics.Address, prev bookkeeping.Block, roundTxnCnt uint64, bootstrappedNet *netState, csParams config.ConsensusParams) (bookkeeping.Block, error) {
func createBlock(src basics.Address, prev bookkeeping.Block, roundTxnCnt uint64, bootstrappedNet *netState, csParams config.ConsensusParams, log logging.Logger) (bookkeeping.Block, error) {
payset := make([]transactions.SignedTxnInBlock, 0, roundTxnCnt)
txibs := make([]transactions.SignedTxnInBlock, 0, roundTxnCnt)

Expand Down Expand Up @@ -560,15 +565,17 @@ func createBlock(src basics.Address, prev bookkeeping.Block, roundTxnCnt uint64,
return bookkeeping.Block{}, err
}

log.Infof("created block[%d] %d txns", block.BlockHeader.Round, len(payset))

return block, nil
}

func generateAccounts(src basics.Address, roundTxnCnt uint64, prev bookkeeping.Block, l *ledger.Ledger, bootstrappedNet *netState, csParams config.ConsensusParams) error {
func generateAccounts(src basics.Address, roundTxnCnt uint64, prev bookkeeping.Block, l *ledger.Ledger, bootstrappedNet *netState, csParams config.ConsensusParams, log logging.Logger) error {

for !bootstrappedNet.accountsCreated {
//create accounts
bootstrappedNet.round++
blk, _ := createBlock(src, prev, roundTxnCnt, bootstrappedNet, csParams)
blk, _ := createBlock(src, prev, roundTxnCnt, bootstrappedNet, csParams, log)
err := l.AddBlock(blk, agreement.Certificate{Round: bootstrappedNet.round})
if err != nil {
fmt.Printf("Error %v\n", err)
Expand Down Expand Up @@ -627,20 +634,22 @@ func accountsNeeded(appsCount uint64, assetCount uint64, params config.Consensus
func createSignedTx(src basics.Address, round basics.Round, params config.ConsensusParams, bootstrappedNet *netState) ([]transactions.SignedTxn, error) {

if bootstrappedNet.nApplications == 0 && bootstrappedNet.nAccounts == 0 && bootstrappedNet.nAssets == 0 {
if !bootstrappedNet.accountsCreated {
bootstrappedNet.log.Infof("done creating accounts, have %d", len(bootstrappedNet.accounts))
}
bootstrappedNet.accountsCreated = true
}
var sgtxns []transactions.SignedTxn

header := transactions.Header{
Fee: basics.MicroAlgos{Raw: 1},
Fee: basics.MicroAlgos{Raw: params.MinTxnFee},
FirstValid: round,
LastValid: round,
GenesisID: bootstrappedNet.genesisID,
GenesisHash: bootstrappedNet.genesisHash,
}

if bootstrappedNet.txnState == protocol.PaymentTx {
var accounts []basics.Address
bootstrappedNet.appsPerAcct = 0
bootstrappedNet.assetPerAcct = 0
n := bootstrappedNet.nAccounts
Expand All @@ -652,7 +661,7 @@ func createSignedTx(src basics.Address, round basics.Round, params config.Consen
for i := uint64(0); i < n; i++ {
secretDst := keypair()
dst := basics.Address(secretDst.SignatureVerifier)
accounts = append(accounts, dst)
bootstrappedNet.accounts = append(bootstrappedNet.accounts, dst)

header.Sender = src

Expand All @@ -668,16 +677,17 @@ func createSignedTx(src basics.Address, round basics.Round, params config.Consen
sgtxns = append(sgtxns, t)
}
bootstrappedNet.nAccounts -= uint64(len(sgtxns))
bootstrappedNet.accounts = accounts
if bootstrappedNet.nAssets > 0 {
bootstrappedNet.log.Info("switch to acfg mode")
bootstrappedNet.txnState = protocol.AssetConfigTx
} else if bootstrappedNet.nApplications > 0 {
bootstrappedNet.log.Info("switch to app cfg mode")
bootstrappedNet.txnState = protocol.ApplicationCallTx
}
} else {
//send payments to created accounts randomly
accti := rand.Intn(len(bootstrappedNet.accounts))
for i := uint64(0); i < n; i++ {
accti := rand.Intn(len(bootstrappedNet.accounts))
header.Sender = src
tx := transactions.Transaction{
Type: protocol.PaymentTx,
Expand All @@ -687,6 +697,8 @@ func createSignedTx(src basics.Address, round basics.Round, params config.Consen
Amount: basics.MicroAlgos{Raw: 0},
},
}
tx.Header.Note = make([]byte, 8)
binary.LittleEndian.PutUint64(tx.Header.Note, bootstrappedNet.roundTxnCnt+i)
t := transactions.SignedTxn{Txn: tx}
sgtxns = append(sgtxns, t)
}
Expand Down Expand Up @@ -731,8 +743,10 @@ func createSignedTx(src basics.Address, round basics.Round, params config.Consen
}
if bootstrappedNet.nAssets == 0 || bootstrappedNet.assetPerAcct == maxAssets {
if bootstrappedNet.nApplications > 0 {
bootstrappedNet.log.Info("switch to app cfg mode")
bootstrappedNet.txnState = protocol.ApplicationCallTx
} else {
bootstrappedNet.log.Info("switch to pay mode")
bootstrappedNet.txnState = protocol.PaymentTx
}

Expand Down Expand Up @@ -785,6 +799,7 @@ func createSignedTx(src basics.Address, round basics.Round, params config.Consen
maxApps = config.Consensus[protocol.ConsensusV30].MaxAppsCreated
}
if bootstrappedNet.nApplications == 0 || bootstrappedNet.appsPerAcct == maxApps {
bootstrappedNet.log.Info("switch to pay mode")
bootstrappedNet.txnState = protocol.PaymentTx
}
}
Expand Down
25 changes: 18 additions & 7 deletions netdeploy/remote/deployedNetwork_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,12 @@ import (

"github.com/algorand/go-algorand/config"
"github.com/algorand/go-algorand/data/basics"
"github.com/algorand/go-algorand/logging"
"github.com/algorand/go-algorand/protocol"
"github.com/algorand/go-algorand/test/partitiontest"
)

func TestCreateSignedTx(t *testing.T) {
func TestCreateSignedTxBasic(t *testing.T) {
partitiontest.PartitionTest(t)

var networkState netState
Expand All @@ -36,6 +37,7 @@ func TestCreateSignedTx(t *testing.T) {
networkState.nAccounts = 10
networkState.roundTxnCnt = 4
networkState.txnState = protocol.PaymentTx
networkState.log = logging.TestingLog(t)

params := config.Consensus[protocol.ConsensusCurrentVersion]

Expand All @@ -50,7 +52,8 @@ func TestCreateSignedTx(t *testing.T) {
require.Equal(t, protocol.PaymentTx, sntx.Txn.Type)
}

initialAccounts := networkState.accounts
initialAccounts := make([]basics.Address, len(networkState.accounts))
copy(initialAccounts, networkState.accounts)

// should be creating assets next
sgtxns, _ = createSignedTx(src, basics.Round(1), params, &networkState)
Expand All @@ -59,7 +62,7 @@ func TestCreateSignedTx(t *testing.T) {
require.Equal(t, protocol.ApplicationCallTx, networkState.txnState)
require.Equal(t, uint64(0), networkState.nAssets)
// same accounts should be used
require.Equal(t, initialAccounts[0], accounts[0])
require.Equal(t, initialAccounts, accounts)
for _, sntx := range sgtxns {
require.Equal(t, protocol.AssetConfigTx, sntx.Txn.Type)
}
Expand All @@ -69,7 +72,7 @@ func TestCreateSignedTx(t *testing.T) {
require.Equal(t, 2, len(sgtxns))
require.Equal(t, protocol.PaymentTx, networkState.txnState)
require.Equal(t, uint64(0), networkState.nApplications)
require.Equal(t, initialAccounts[0], accounts[0])
require.Equal(t, initialAccounts, accounts)
for _, sntx := range sgtxns {
require.Equal(t, protocol.ApplicationCallTx, sntx.Txn.Type)
}
Expand All @@ -78,14 +81,22 @@ func TestCreateSignedTx(t *testing.T) {
sgtxns, _ = createSignedTx(src, basics.Round(1), params, &networkState)
require.Equal(t, 4, len(sgtxns))
require.Equal(t, protocol.PaymentTx, networkState.txnState)
//new accounts should be created
accounts = networkState.accounts
require.NotEqual(t, initialAccounts[0], accounts[0])
require.Equal(t, initialAccounts, accounts)
for _, sntx := range sgtxns {
require.Equal(t, protocol.PaymentTx, sntx.Txn.Type)
}
}

func TestCreateSignedTxAssets(t *testing.T) {
// assets per account should not exceed limit
partitiontest.PartitionTest(t)

params := config.Consensus[protocol.ConsensusCurrentVersion]
secretDst := keypair()
src := basics.Address(secretDst.SignatureVerifier)

var networkState netState
networkState.log = logging.TestingLog(t)
networkState.txnState = protocol.PaymentTx
networkState.nAssets = 10
networkState.nApplications = 10
Expand Down
Loading