From c486d6c402f9c25b39b7f111acbe31ea202fc709 Mon Sep 17 00:00:00 2001 From: SergioDemianLerner Date: Wed, 24 Apr 2019 22:43:51 -0300 Subject: [PATCH] hundreds of state tests added again. --- .../stSystemOperationsTestFiller.json | 2 +- .../co/rsk/jsontestsuite/LocalStateTest.java | 205 +++++++++++++++--- .../jsontestsuite/GitHubJSONTestSuite.java | 3 +- .../ethereum/jsontestsuite/TestRunner.java | 37 +++- .../runners/StateTestRunner.java | 25 ++- .../validators/AccountValidator.java | 29 ++- .../validators/BlockHeaderValidator.java | 28 +-- .../validators/LogsValidator.java | 12 +- .../validators/OutputValidator.java | 4 +- .../validators/RepositoryValidator.java | 27 ++- .../validators/ValidationStats.java | 15 ++ .../StateTests/stSystemOperationsTest.json | 4 +- 12 files changed, 294 insertions(+), 97 deletions(-) create mode 100644 rskj-core/src/test/java/org/ethereum/jsontestsuite/validators/ValidationStats.java diff --git a/rskj-core/TestGeneration/libethereum/StateTestsFiller/stSystemOperationsTestFiller.json b/rskj-core/TestGeneration/libethereum/StateTestsFiller/stSystemOperationsTestFiller.json index b5ddf539ff4..59bd68d8805 100644 --- a/rskj-core/TestGeneration/libethereum/StateTestsFiller/stSystemOperationsTestFiller.json +++ b/rskj-core/TestGeneration/libethereum/StateTestsFiller/stSystemOperationsTestFiller.json @@ -86,7 +86,7 @@ "createWithInvalidOpcode": { "env" : { "currentCoinbase" : "2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", - "currentDifficulty" : "115792089237316195423570985008687907853269984665640564039457584007913129639935", + "currentDifficulty" : "0x1fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "currentGasLimit" : "1000000", "currentNumber" : "2463000", "currentTimestamp" : "2", diff --git a/rskj-core/src/test/java/co/rsk/jsontestsuite/LocalStateTest.java b/rskj-core/src/test/java/co/rsk/jsontestsuite/LocalStateTest.java index efc95bf5b50..770b3b1ef13 100644 --- a/rskj-core/src/test/java/co/rsk/jsontestsuite/LocalStateTest.java +++ b/rskj-core/src/test/java/co/rsk/jsontestsuite/LocalStateTest.java @@ -15,6 +15,23 @@ * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see . */ +/* + * This file is part of RskJ + * Copyright (C) 2017 RSK Labs Ltd. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ package co.rsk.jsontestsuite; @@ -42,7 +59,7 @@ public class LocalStateTest { - @Test // this method is mostly for hands-on convenient testing + @Ignore // this method is mostly for hands-on convenient testing public void stSingleTest() throws ParseException, IOException { String json = getJSON("stSystemOperationsTest"); GitHubJSONTestSuite.runStateTest(json, "suicideSendEtherPostDeath"); @@ -61,6 +78,7 @@ public void stCallCodes() throws ParseException, IOException { Set excluded = new HashSet<>(); String json = getJSON("stCallCodes"); + /* Recursive tests excluded */ excluded.add("callcodecallcode_11"); excluded.add("callcodecallcode_11"); excluded.add("callcodecallcallcode_101"); @@ -83,7 +101,7 @@ public void stCallCodes() throws ParseException, IOException { excluded.add("callcodecallcall_ABCB_RECURSIVE"); excluded.add("callcallcodecallcode_ABCB_RECURSIVE"); excluded.add("callcodecallcallcode_ABCB_RECURSIVE"); - + /* */ GitHubJSONTestSuite.runStateTest(json, excluded); @@ -94,21 +112,26 @@ public void stCallDelegateCodes() throws ParseException, IOException { Set excluded = new HashSet<>(); String json = getJSON("stCallDelegateCodes"); + + // Recursive tests excluded */ excluded.add("callcodecallcodecallcode_ABCB_RECURSIVE"); - excluded.add("callcodecallcallcode_101"); excluded.add("callcodecallcodecall_ABCB_RECURSIVE"); excluded.add("callcallcodecall_ABCB_RECURSIVE"); + excluded.add("callcallcallcode_ABCB_RECURSIVE"); + excluded.add("callcodecallcall_ABCB_RECURSIVE"); + excluded.add("callcallcodecallcode_ABCB_RECURSIVE"); + excluded.add("callcodecallcallcode_ABCB_RECURSIVE"); + + /* These tests would fail if balances are checked + excluded.add("callcodecallcallcode_101"); excluded.add("callcallcodecallcode_011"); excluded.add("callcodecallcall_100"); excluded.add("callcallcodecall_010"); - excluded.add("callcallcallcode_ABCB_RECURSIVE"); - excluded.add("callcodecallcall_ABCB_RECURSIVE"); excluded.add("callcallcode_01"); excluded.add("callcodecallcodecall_110"); excluded.add("callcallcallcode_001"); - excluded.add("callcallcodecallcode_ABCB_RECURSIVE"); excluded.add("callcodecall_10"); - excluded.add("callcodecallcallcode_ABCB_RECURSIVE"); + */ GitHubJSONTestSuite.runStateTest(json, excluded); } @@ -117,22 +140,27 @@ public void stCallDelegateCodesCallCode() throws ParseException, IOException { Set excluded = new HashSet<>(); String json = getJSON("stCallDelegateCodesCallCode"); + + // Recursive tests excluded excluded.add("callcodecallcodecallcode_ABCB_RECURSIVE"); - excluded.add("callcodecallcallcode_101"); excluded.add("callcodecallcodecall_ABCB_RECURSIVE"); excluded.add("callcallcodecall_ABCB_RECURSIVE"); + excluded.add("callcallcallcode_ABCB_RECURSIVE"); + excluded.add("callcodecallcall_ABCB_RECURSIVE"); + excluded.add("callcallcodecallcode_ABCB_RECURSIVE"); + excluded.add("callcodecallcallcode_ABCB_RECURSIVE"); + + /* These tests would fail if balances are checked + excluded.add("callcodecallcallcode_101"); excluded.add("callcallcodecallcode_011"); excluded.add("callcodecallcall_100"); excluded.add("callcallcodecall_010"); - excluded.add("callcallcallcode_ABCB_RECURSIVE"); excluded.add("callcodecallcodecallcode_111_SuicideEnd"); - excluded.add("callcodecallcall_ABCB_RECURSIVE"); excluded.add("callcallcode_01"); excluded.add("callcodecallcodecall_110"); excluded.add("callcallcallcode_001"); - excluded.add("callcallcodecallcode_ABCB_RECURSIVE"); excluded.add("callcodecall_10"); - excluded.add("callcodecallcallcode_ABCB_RECURSIVE"); + */ GitHubJSONTestSuite.runStateTest(json, excluded); } @@ -141,8 +169,11 @@ public void stHomeSteadSpecific() throws ParseException, IOException { Set excluded = new HashSet<>(); String json = getJSON("stHomeSteadSpecific"); + + /* These tests would fail if balances are checked excluded.add("contractCreationOOGdontLeaveEmptyContract"); excluded.add("createContractViaContractOOGInitCode"); + */ GitHubJSONTestSuite.runStateTest(json, excluded); } @@ -151,6 +182,9 @@ public void stCallCreateCallCodeTest() throws ParseException, IOException { Set excluded = new HashSet<>(); String json = getJSON("stCallCreateCallCodeTest"); + // *** java.lang.instrument ASSERTION FAILED ***: "!errorOutstanding" with message transform method call failed at JPLISAgent.c line: 844 + + /* Recursive tests excluded */ excluded.add("Callcode1024OOG"); excluded.add("Call1024PreCalls"); excluded.add("callWithHighValueOOGinCall"); @@ -168,7 +202,7 @@ public void stCallCreateCallCodeTest() throws ParseException, IOException { excluded.add("callcodeWithHighValue"); excluded.add("createInitFailBadJumpDestination"); excluded.add("callcodeWithHighValueAndGasOOG"); - + /* */ GitHubJSONTestSuite.runStateTest(json, excluded); } @@ -177,12 +211,15 @@ public void stCallCreateCallCodeTest() throws ParseException, IOException { public void stDelegatecallTest() throws ParseException, IOException { Set excluded = new HashSet<>(); String json = getJSON("stDelegatecallTest"); + + // All these tests use recursive calls and therefore are incompatible excluded.add("Delegatecall1024OOG"); excluded.add("Call1024PreCalls"); excluded.add("Call1024BalanceTooLow"); excluded.add("CallRecursiveBombPreCall"); excluded.add("Call1024OOG"); excluded.add("Delegatecall1024"); + GitHubJSONTestSuite.runStateTest(json, excluded); } @@ -190,9 +227,24 @@ public void stDelegatecallTest() throws ParseException, IOException { public void stInitCodeTest() throws ParseException, IOException { Set excluded = new HashSet<>(); String json = getJSON("stInitCodeTest"); + + /* These tests require not to check account balances excluded.add("CallContractToCreateContractWhichWouldCreateContractIfCalled"); excluded.add("CallContractToCreateContractOOGBonusGas"); - excluded.add("CallRecursiveContract"); + */ + // CallRecursiveContract must be excluded because it creates + // contracts recursively and the number of contracts to create + // is given by the amount of gas (it calls itself until + // CALL fails with OOG). + // Since CALL works differently between RSK and Ethereum, the test fails. + // + // "code": "{[[ 2 ]](ADDRESS)(CODECOPY 0 0 32)(CREATE 0 0 32)}", + // SSTORE[2] ADDRESS + // CODECOPY (to:0 from:0 length:32) + // CREATE a contract (Value=0 InOffset =0 InSize =32) + // This creates a CLONE of the contract + excluded.add("CallRecursiveContract"); + GitHubJSONTestSuite.runStateTest(json, excluded); } @@ -201,6 +253,15 @@ public void stInitCodeTest() throws ParseException, IOException { public void stLogTests() throws ParseException, IOException { Set excluded = new HashSet<>(); String json = getJSON("stLogTests"); + + + /* All these tests use CALL to a contract that executes LOG + * Because CALL consumes a different amount of gas compared to Ethereum + * all these tests will fail. + * To enable them, we indicate not to check balances. + */ + + /* These tests would fail if balances are checked excluded.add("log3_nonEmptyMem"); excluded.add("log1_emptyMem"); excluded.add("log2_nonEmptyMem"); @@ -247,7 +308,9 @@ public void stLogTests() throws ParseException, IOException { excluded.add("log4_nonEmptyMem_logMemSize1"); excluded.add("log0_nonEmptyMem"); excluded.add("log3_logMemsizeZero"); + */ GitHubJSONTestSuite.runStateTest(json, excluded); + /* */ } @Test @@ -255,6 +318,7 @@ public void stPreCompiledContracts() throws ParseException, IOException { Set excluded = new HashSet<>(); String json = getJSON("stPreCompiledContracts"); + /* These tests would fail if balances are checked excluded.add("CALLCODEEcrecover0_0input"); excluded.add("CALLCODEIdentity_1_nonzeroValue"); excluded.add("CallEcrecover80"); @@ -287,16 +351,33 @@ public void stPreCompiledContracts() throws ParseException, IOException { excluded.add("CALLCODEEcrecoverS_prefixed0"); excluded.add("CALLCODEEcrecover0_NoGas"); excluded.add("CallEcrecover0_0input"); + */ GitHubJSONTestSuite.runStateTest(json, excluded); } @Test public void stMemoryStressTest() throws ParseException, IOException { Set excluded = new HashSet<>(); - excluded.add("mload32bitBound_return2");// The test extends memory to 4Gb which can't be handled with Java arrays - excluded.add("mload32bitBound_return"); // The test extends memory to 4Gb which can't be handled with Java arrays - excluded.add("mload32bitBound_Msize"); // The test extends memory to 4Gb which can't be handled with Java arrays + + /* These tests would fail if balances are checked + excluded.add("mload32bitBound_return2"); + excluded.add("mload32bitBound_return"); + */ + + // mload32bitBound_Msize has to be excluded because RSK does not + // allow access to addres 4294967295 (0xFFFFFFFF). + // "code" : "{ [4294967295] 1 [[ 0 ]] (MSIZE)} ", + // This compiles to: + // PUSH 4294967295 + // PUSH 1 + // MSTORE (stores 1 at 4294967295, fails in RSK) + // PUSH 0 + // PUSH MSIZE + // SSTORE (Stores MSIZE at persistent cell 0) + excluded.add("mload32bitBound_Msize"); // Tries to store something in address 4294967295. This causes OOG in RSK + String json = getJSON("stMemoryStressTest"); + GitHubJSONTestSuite.runStateTest(json, excluded); } @@ -308,13 +389,19 @@ public void stMemoryTest() throws ParseException, IOException { GitHubJSONTestSuite.runStateTest(json, excluded); } - @Test + @Ignore + // While RSK passes these tests, they have no "expect" clauses. Nothing is checked + // after each test is finished. I suppose these tests serve only for checking + // the performance of the VM. However there are no time contrains here, so + // the tests are currenlty useless. + public void stQuadraticComplexityTest() throws ParseException, IOException { Set excluded = new HashSet<>(); String json = getJSON("stQuadraticComplexityTest"); + // The test Call1MB1024Calldepth must be excluded because RSK doesn't + // has the 1024 call depth limit, while Ethereum uses a 63/64 gas spending + // per CALL to prevent hitting the limit. excluded.add("Call1MB1024Calldepth"); - excluded.add("Call50000_sha256"); - GitHubJSONTestSuite.runStateTest(json, excluded); } @@ -322,6 +409,8 @@ public void stQuadraticComplexityTest() throws ParseException, IOException { public void stSolidityTest() throws ParseException, IOException { Set excluded = new HashSet<>(); String json = getJSON("stSolidityTest"); + + /* These tests would fail if balances are checked excluded.add("TestBlockAndTransactionProperties"); excluded.add("TestCryptographicFunctions"); excluded.add("TestStructuresAndVariabless"); @@ -334,6 +423,7 @@ public void stSolidityTest() throws ParseException, IOException { excluded.add("CallLowLevelCreatesSolidity"); excluded.add("RecursiveCreateContractsCreate4Contracts"); excluded.add("ContractInheritance"); + */ GitHubJSONTestSuite.runStateTest(json, excluded); } @@ -342,6 +432,8 @@ public void stRecursiveCreate() throws ParseException, IOException { Set excluded = new HashSet<>(); String json = getJSON("stRecursiveCreate"); + + // Recursive tests excluded excluded.add("recursiveCreateReturnValue"); excluded.add("recursiveCreate"); excluded.add("testRandomTest"); @@ -354,10 +446,13 @@ public void stRefundTest() throws ParseException, IOException { Set excluded = new HashSet<>(); String json = getJSON("stRefundTest"); + + /* These tests would fail if balances are checked excluded.add("refund_singleSuicide"); excluded.add("refund_multimpleSuicide"); excluded.add("refund600"); excluded.add("refund50percentCap"); + */ GitHubJSONTestSuite.runStateTest(json, excluded); } @@ -366,9 +461,13 @@ public void stSpecialTest() throws ParseException, IOException { Set excluded = new HashSet<>(); String json = getJSON("stSpecialTest"); - excluded.add("makeMoney"); + // Why ? Recursion ? excluded.add("JUMPDEST_Attack"); excluded.add("JUMPDEST_AttackwithJump"); + /* These tests would fail if balances are checked + excluded.add("makeMoney"); + + */ GitHubJSONTestSuite.runStateTest(json, excluded); } @@ -377,8 +476,10 @@ public void stBlockHashTest() throws ParseException, IOException { String json = getJSON("stBlockHashTest"); Set excluded = new HashSet<>(); + /* These tests would fail if balances are checked excluded.add("blockhash0"); excluded.add("blockhashDOS-sec71"); + */ GitHubJSONTestSuite.runStateTest(json,excluded); } @@ -387,10 +488,32 @@ public void stSystemOperationsTest() throws IOException { Set excluded = new HashSet<>(); String json = getJSON("stSystemOperationsTest"); - excluded.add("createWithInvalidOpcode"); + + // Recursion in the Unit test framework does not work well + // They throw an internal exception due to stack overflow. excluded.add("CallRecursiveBombLog2"); excluded.add("CallRecursiveBombLog"); excluded.add("CallRecursiveBomb0_OOG_atMaxCallDepth"); + excluded.add("CallRecursiveBomb3"); + excluded.add("CallRecursiveBomb2"); + excluded.add("CallRecursiveBomb1"); + excluded.add("CallRecursiveBomb0"); + excluded.add("ABAcallsSuicide1"); + excluded.add("ABAcalls0"); + excluded.add("ABAcalls1"); + excluded.add("ABAcalls2"); + excluded.add("ABAcalls3"); + + // createWithInvalidOpcode and testRandomTest tests: + // This test fails because it specifies a header with + // an invalid block difficulty: A block difficulty must be positive or zero + // This is because the test uses the difficulty 2^256-1 and this is interpreted as + // a negative number by RLP.parseBlockDifficulty(). + // This was solved by changing the first hex digit of the difficulty from "f" to "1". + // Same for testRandomTest + // + + /* These tests would fail if balances are checked excluded.add("testRandomTest"); excluded.add("callcodeToNameRegistratorAddresTooBigLeft"); excluded.add("callcodeTo0"); @@ -403,20 +526,13 @@ public void stSystemOperationsTest() throws IOException { excluded.add("ABAcallsSuicide0"); excluded.add("CallToNameRegistratorNotMuchMemory1"); excluded.add("CallToNameRegistratorOutOfGas"); - excluded.add("ABAcallsSuicide1"); - excluded.add("ABAcalls1"); - excluded.add("ABAcalls2"); - excluded.add("ABAcalls3"); excluded.add("CallToNameRegistrator0"); - excluded.add("ABAcalls0"); - excluded.add("CallRecursiveBomb3"); - excluded.add("CallRecursiveBomb2"); excluded.add("CallToNameRegistratorAddressTooBigRight"); - excluded.add("CallRecursiveBomb1"); - excluded.add("CallRecursiveBomb0"); excluded.add("CallToNameRegistratorAddressTooBigLeft"); excluded.add("CallToReturn1"); excluded.add("CallToNameRegistratorZeorSizeMemExpansion"); + */ + GitHubJSONTestSuite.runStateTest(json, excluded); } @@ -424,6 +540,13 @@ public void stSystemOperationsTest() throws IOException { public void stTransactionTest() throws ParseException, IOException { Set excluded = new HashSet<>(); String json = getJSON("stTransactionTest"); + + //StoreGasOnCreate must be excluded because + //CREATE seems to consume more gas in RSK than in Ethereum. + + excluded.add("StoreGasOnCreate"); + + /* These tests would fail if balances are checked excluded.add("TransactionNonceCheck2"); excluded.add("TransactionNonceCheck"); excluded.add("InternalCallHittingGasLimit2"); @@ -431,10 +554,11 @@ public void stTransactionTest() throws ParseException, IOException { excluded.add("InternlCallStoreClearsOOG"); excluded.add("StoreClearsAndInternlCallStoreClearsOOG"); excluded.add("InternalCallHittingGasLimitSuccess"); - excluded.add("StoreGasOnCreate"); + excluded.add("SuicidesAndInternlCallSuicidesSuccess"); excluded.add("InternlCallStoreClearsSucces"); excluded.add("StoreClearsAndInternlCallStoreClearsSuccess"); + */ GitHubJSONTestSuite.runStateTest(json, excluded); } @@ -445,10 +569,14 @@ public void stTransitionTest() throws ParseException, IOException { String json = getJSON("stTransitionTest"); excluded.add("createNameRegistratorPerTxsNotEnoughGasBefore"); - excluded.add("delegatecallAtTransition"); + // Investigate... excluded.add("delegatecallBeforeTransition"); + + /* These tests would fail if balances are checked + excluded.add("delegatecallAtTransition"); excluded.add("delegatecallAfterTransition"); excluded.add("createNameRegistratorPerTxsBefore"); + */ GitHubJSONTestSuite.runStateTest(json, excluded); } @@ -458,6 +586,12 @@ public void stWalletTest() throws ParseException, IOException { Set excluded = new HashSet<>(); String json = getJSON("stWalletTest"); + + // Investigate reason + excluded.add("dayLimitConstruction"); + excluded.add("walletConstruction"); + + /* These tests would fail if balances are checked excluded.add("walletExecuteUnderDailyLimit"); excluded.add("multiOwnedConstructionCorrect"); excluded.add("walletChangeRequirementRemovePendingTransaction"); @@ -470,7 +604,7 @@ public void stWalletTest() throws ParseException, IOException { excluded.add("multiOwnedChangeRequirementTo0"); excluded.add("dayLimitSetDailyLimit"); excluded.add("multiOwnedChangeRequirementTo1"); - excluded.add("walletConstruction"); + excluded.add("multiOwnedChangeRequirementTo2"); excluded.add("multiOwnedIsOwnerTrue"); excluded.add("walletChangeOwnerRemovePendingTransaction"); @@ -478,7 +612,7 @@ public void stWalletTest() throws ParseException, IOException { excluded.add("multiOwnedChangeOwnerNoArguments"); excluded.add("dayLimitSetDailyLimitNoData"); excluded.add("walletExecuteOverDailyLimitMultiOwner"); - excluded.add("dayLimitConstruction"); + excluded.add("multiOwnedAddOwnerAddMyself"); excluded.add("multiOwnedAddOwner"); excluded.add("walletKill"); @@ -493,6 +627,7 @@ public void stWalletTest() throws ParseException, IOException { excluded.add("walletKillNotByOwner"); excluded.add("walletKillToWallet"); excluded.add("walletExecuteOverDailyLimitOnlyOneOwner"); + */ GitHubJSONTestSuite.runStateTest(json, excluded); } diff --git a/rskj-core/src/test/java/org/ethereum/jsontestsuite/GitHubJSONTestSuite.java b/rskj-core/src/test/java/org/ethereum/jsontestsuite/GitHubJSONTestSuite.java index a397cb15222..c4abc056033 100644 --- a/rskj-core/src/test/java/org/ethereum/jsontestsuite/GitHubJSONTestSuite.java +++ b/rskj-core/src/test/java/org/ethereum/jsontestsuite/GitHubJSONTestSuite.java @@ -263,8 +263,9 @@ public static void runStateTest(String jsonSuite, Set excluded) throws I List result = StateTestRunner.run(testCases.get(testName)); - if (!result.isEmpty()) + if (!result.isEmpty()) { summary.put(testName, false); + } else summary.put(testName, true); } diff --git a/rskj-core/src/test/java/org/ethereum/jsontestsuite/TestRunner.java b/rskj-core/src/test/java/org/ethereum/jsontestsuite/TestRunner.java index c843d7a464a..9535b344589 100644 --- a/rskj-core/src/test/java/org/ethereum/jsontestsuite/TestRunner.java +++ b/rskj-core/src/test/java/org/ethereum/jsontestsuite/TestRunner.java @@ -44,6 +44,7 @@ import org.ethereum.jsontestsuite.model.TransactionTck; import org.ethereum.jsontestsuite.validators.BlockHeaderValidator; import org.ethereum.jsontestsuite.validators.RepositoryValidator; +import org.ethereum.jsontestsuite.validators.ValidationStats; import org.ethereum.listener.CompositeEthereumListener; import org.ethereum.listener.TestCompositeEthereumListener; import org.ethereum.util.ByteUtil; @@ -86,6 +87,23 @@ public class TestRunner { private ProgramTrace trace = null; private boolean setNewStateRoot; private boolean validateGasUsed = false; // until EIP150 test cases are ready. + private boolean validateBalances = true; + private boolean validateStateRoots =false; + + public TestRunner setValidateGasUsed( boolean v) { + validateGasUsed= v; + return this; + } + + public TestRunner setValidateStateRoots ( boolean v) { + validateStateRoots = v; + return this; + } + + public TestRunner setValidateBalances(boolean v) { + validateBalances = v; + return this; + } public List runTestSuite(TestSuite testSuite) { @@ -107,6 +125,8 @@ public List runTestSuite(TestSuite testSuite) { public List runTestCase(BlockTestCase testCase) { /* 1 */ // Create genesis + init pre state + ValidationStats vStats = new ValidationStats(); + Block genesis = build(testCase.getGenesisBlockHeader(), null, null); Repository repository = RepositoryBuilder.build(testCase.getPre()); @@ -165,7 +185,7 @@ public List runTestCase(BlockTestCase testCase) { tBlock = blockFactory.decodeBlock(rlp); ArrayList outputSummary = - BlockHeaderValidator.valid(tBlock.getHeader(), block.getHeader()); + BlockHeaderValidator.valid(tBlock.getHeader(), block.getHeader(),null); if (!outputSummary.isEmpty()){ for (String output : outputSummary) @@ -193,17 +213,16 @@ public List runTestCase(BlockTestCase testCase) { byte[] bestHash = Hex.decode(testCase.getLastblockhash()); String finalRoot = Hex.toHexString(blockStore.getBlockByHash(bestHash).getStateRoot()); - /* - if (!blockchain.byTest) // If this comes from ETH, it won't match - if (!finalRoot.equals(currRoot)){ - String formattedString = String.format("Root hash doesn't match best: expected: %s current: %s", - finalRoot, currRoot); - results.add(formattedString); + if (validateStateRoots) { + if (!finalRoot.equals(currRoot)) { + String formattedString = String.format("Root hash doesn't match best: expected: %s current: %s", + finalRoot, currRoot); + results.add(formattedString); + } } - */ Repository postRepository = RepositoryBuilder.build(testCase.getPostState()); - List repoResults = RepositoryValidator.valid(repository, postRepository, false /*!blockchain.byTest*/); + List repoResults = RepositoryValidator.valid(repository, postRepository, validateStateRoots,validateBalances,null); results.addAll(repoResults); return results; diff --git a/rskj-core/src/test/java/org/ethereum/jsontestsuite/runners/StateTestRunner.java b/rskj-core/src/test/java/org/ethereum/jsontestsuite/runners/StateTestRunner.java index f56475f80db..9821c63e73e 100644 --- a/rskj-core/src/test/java/org/ethereum/jsontestsuite/runners/StateTestRunner.java +++ b/rskj-core/src/test/java/org/ethereum/jsontestsuite/runners/StateTestRunner.java @@ -38,6 +38,7 @@ import org.ethereum.jsontestsuite.validators.LogsValidator; import org.ethereum.jsontestsuite.validators.OutputValidator; import org.ethereum.jsontestsuite.validators.RepositoryValidator; +import org.ethereum.jsontestsuite.validators.ValidationStats; import org.ethereum.listener.EthereumListenerAdapter; import org.ethereum.util.ByteUtil; import org.ethereum.vm.LogInfo; @@ -74,6 +75,7 @@ public static List run(StateTestCase stateTestCase2) { protected Env env; protected ProgramInvokeFactory invokeFactory; protected Block block; + protected ValidationStats vStats; // Enable when State tests are changed to support paying // the fees into REMASC instead that into the coinbase account @@ -128,7 +130,7 @@ protected ProgramResult executeTransaction(Transaction tx) { } public List runImpl() { - + vStats = new ValidationStats(); logger.info(""); repository = RepositoryBuilder.build(stateTestCase.getPre()); logger.info("loaded repository"); @@ -180,14 +182,16 @@ public List runImpl() { List origLogs = programResult.getLogInfoList(); List postLogs = LogBuilder.build(stateTestCase.getLogs()); - List logsResult = LogsValidator.valid(origLogs, postLogs); + List logsResult = LogsValidator.valid(origLogs, postLogs,vStats); Repository postRepository = RepositoryBuilder.build(stateTestCase.getPost()); - List repoResults = RepositoryValidator.valid(repository, postRepository, false /*!blockchain.byTest*/); + + // Balances cannot be validated because has consumption for CALLs differ. + List repoResults = RepositoryValidator.valid(repository, postRepository, false ,false,vStats); logger.info("--------- POST Validation---------"); List outputResults = - OutputValidator.valid(Hex.toHexString(programResult.getHReturn()), stateTestCase.getOut()); + OutputValidator.valid(Hex.toHexString(programResult.getHReturn()), stateTestCase.getOut(),vStats); List results = new ArrayList<>(); results.addAll(repoResults); @@ -198,14 +202,25 @@ public List runImpl() { logger.error(result); } + if ((vStats.storageChecks==0) && (vStats.logChecks==0) && + (vStats.balancetChecks==0) && (vStats.outputChecks==0) && + (vStats.blockChecks==0)) { + // This generally mean that the test didn't check anything + // AccountChecks are considered not indicative of the result of the test + logger.info("IRRELEVANT\n"); + } logger.info("\n\n"); return results; } + public static final byte[] ZERO32_BYTE_ARRAY = new byte[32]; public Block build(Env env) { return new Block( blockFactory.newHeader( - ByteUtil.EMPTY_BYTE_ARRAY, ByteUtil.EMPTY_BYTE_ARRAY, env.getCurrentCoinbase(), + // Don't use the empty parent hash because it's used to log and + // when log entries are printed with empty parent hash it throws + // an exception. + ZERO32_BYTE_ARRAY , ByteUtil.EMPTY_BYTE_ARRAY, env.getCurrentCoinbase(), EMPTY_TRIE_HASH, EMPTY_TRIE_HASH, EMPTY_TRIE_HASH, ByteUtil.EMPTY_BYTE_ARRAY, env.getCurrentDifficulty(), byteArrayToLong(env.getCurrentNumber()), env.getCurrentGasLimit(), 0L, byteArrayToLong(env.getCurrentTimestamp()), diff --git a/rskj-core/src/test/java/org/ethereum/jsontestsuite/validators/AccountValidator.java b/rskj-core/src/test/java/org/ethereum/jsontestsuite/validators/AccountValidator.java index 026fe8ba28f..4bc7f6e3067 100644 --- a/rskj-core/src/test/java/org/ethereum/jsontestsuite/validators/AccountValidator.java +++ b/rskj-core/src/test/java/org/ethereum/jsontestsuite/validators/AccountValidator.java @@ -35,12 +35,12 @@ public class AccountValidator { private static final byte[] EMPTY_DATA_HASH = HashUtil.keccak256(EMPTY_BYTE_ARRAY); - public static List valid(RskAddress addr, Repository currentRepository, Repository expectedRepository) { + public static List valid(RskAddress addr, Repository currentRepository, Repository expectedRepository,boolean validateBalance,ValidationStats vStats) { AccountState currentState = currentRepository.getAccountState(addr); AccountState expectedState = expectedRepository.getAccountState(addr); List results = new ArrayList<>(); - + if (vStats!=null) vStats.accountChecks++; if (currentState == null || !currentRepository.isContract(addr)) { String formattedString = String.format("Account: %s: expected but doesn't exist", addr); @@ -48,6 +48,7 @@ public static List valid(RskAddress addr, Repository currentRepository, return results; } + if (vStats!=null) vStats.accountChecks++; if (expectedState == null || !expectedRepository.isContract(addr)) { String formattedString = String.format("Account: %s: unexpected account in the repository", addr); @@ -55,14 +56,17 @@ public static List valid(RskAddress addr, Repository currentRepository, return results; } - - Coin expectedBalance = expectedState.getBalance(); - if (!currentState.getBalance().equals(expectedBalance)) { - String formattedString = String.format("Account: %s: has unexpected balance, expected balance: %s found balance: %s", - addr, expectedBalance.toString(), currentState.getBalance().toString()); - results.add(formattedString); + if (validateBalance) { + if (vStats!=null) vStats.balancetChecks++; + Coin expectedBalance = expectedState.getBalance(); + if (!currentState.getBalance().equals(expectedBalance)) { + String formattedString = String.format("Account: %s: has unexpected balance, expected balance: %s found balance: %s", + addr, expectedBalance.toString(), currentState.getBalance().toString()); + results.add(formattedString); + } } + if (vStats!=null) vStats.accountChecks++; BigInteger expectedNonce = expectedState.getNonce(); if (currentState.getNonce().compareTo(expectedNonce) != 0) { String formattedString = String.format("Account: %s: has unexpected nonce, expected nonce: %s found nonce: %s", @@ -72,6 +76,8 @@ public static List valid(RskAddress addr, Repository currentRepository, byte[] code = Arrays.equals(currentState.getCodeHash(), EMPTY_DATA_HASH) ? new byte[0] : currentRepository.getCode(addr); + + if (vStats!=null) vStats.accountChecks++; if (!Arrays.equals(expectedRepository.getCode(addr), code)) { String formattedString = String.format("Account: %s: has unexpected code, expected code: %s found code: %s", addr, Hex.toHexString(expectedRepository.getCode(addr)), Hex.toHexString(currentRepository.getCode(addr))); @@ -89,6 +95,7 @@ public static List valid(RskAddress addr, Repository currentRepository, DataWord currentValue = currentRepository.getStorageValue(addr, key); DataWord expectedValue = expectedRepository.getStorageValue(addr, key); + if (vStats!=null) vStats.storageChecks++; if (expectedValue == null) { String formattedString = String.format("Account: %s: has unexpected storage data: %s = %s", @@ -99,7 +106,7 @@ public static List valid(RskAddress addr, Repository currentRepository, results.add(formattedString); continue; } - + if (vStats!=null) vStats.storageChecks++; if (!expectedValue.equals(currentValue)) { String formattedString = String.format("Account: %s: has unexpected value, for key: %s , expectedValue: %s real value: %s", @@ -115,9 +122,9 @@ public static List valid(RskAddress addr, Repository currentRepository, while (expectedKeys.hasNext()) { DataWord key = expectedKeys.next(); - + if (vStats!=null) vStats.storageChecks++; if (!checked.contains(key)) { - String formattedString = String.format("Account: %s: doesn't exist expected storage key: %s", + String formattedString = String.format("Account: %s: doesn't exist. Expected storage key: %s", addr, key.toString()); results.add(formattedString); } diff --git a/rskj-core/src/test/java/org/ethereum/jsontestsuite/validators/BlockHeaderValidator.java b/rskj-core/src/test/java/org/ethereum/jsontestsuite/validators/BlockHeaderValidator.java index 39ddbfbc9fd..f3b5aa5cfe6 100644 --- a/rskj-core/src/test/java/org/ethereum/jsontestsuite/validators/BlockHeaderValidator.java +++ b/rskj-core/src/test/java/org/ethereum/jsontestsuite/validators/BlockHeaderValidator.java @@ -29,10 +29,10 @@ public class BlockHeaderValidator { - public static ArrayList valid(BlockHeader orig, BlockHeader valid) { + public static ArrayList valid(BlockHeader orig, BlockHeader valid,ValidationStats vStats) { ArrayList outputSummary = new ArrayList<>(); - + if (vStats!=null) vStats.blockChecks++; if (!orig.getParentHash().equals(valid.getParentHash())) { String output = @@ -43,7 +43,7 @@ public static ArrayList valid(BlockHeader orig, BlockHeader valid) { outputSummary.add(output); } - + if (vStats!=null) vStats.blockChecks++; if (!toHexString(orig.getUnclesHash()) .equals(toHexString(valid.getUnclesHash()))) { @@ -55,7 +55,7 @@ public static ArrayList valid(BlockHeader orig, BlockHeader valid) { outputSummary.add(output); } - + if (vStats!=null) vStats.blockChecks++; if (!orig.getCoinbase().equals(valid.getCoinbase())) { String output = @@ -66,7 +66,7 @@ public static ArrayList valid(BlockHeader orig, BlockHeader valid) { outputSummary.add(output); } - + if (vStats!=null) vStats.blockChecks++; if (!toHexString(orig.getStateRoot()) .equals(toHexString(valid.getStateRoot()))) { @@ -78,7 +78,7 @@ public static ArrayList valid(BlockHeader orig, BlockHeader valid) { outputSummary.add(output); } - + if (vStats!=null) vStats.blockChecks++; if (!toHexString(orig.getTxTrieRoot()) .equals(toHexString(valid.getTxTrieRoot()))) { @@ -90,7 +90,7 @@ public static ArrayList valid(BlockHeader orig, BlockHeader valid) { outputSummary.add(output); } - + if (vStats!=null) vStats.blockChecks++; if (!toHexString(orig.getReceiptsRoot()) .equals(toHexString(valid.getReceiptsRoot()))) { @@ -102,7 +102,7 @@ public static ArrayList valid(BlockHeader orig, BlockHeader valid) { outputSummary.add(output); } - + if (vStats!=null) vStats.blockChecks++; if (!toHexString(orig.getLogsBloom()) .equals(toHexString(valid.getLogsBloom()))) { @@ -114,7 +114,7 @@ public static ArrayList valid(BlockHeader orig, BlockHeader valid) { outputSummary.add(output); } - + if (vStats!=null) vStats.blockChecks++; if (!orig.getDifficulty().equals(valid.getDifficulty())) { String output = @@ -125,7 +125,7 @@ public static ArrayList valid(BlockHeader orig, BlockHeader valid) { outputSummary.add(output); } - + if (vStats!=null) vStats.blockChecks++; if (orig.getTimestamp() != valid.getTimestamp()) { String output = @@ -136,7 +136,7 @@ public static ArrayList valid(BlockHeader orig, BlockHeader valid) { outputSummary.add(output); } - + if (vStats!=null) vStats.blockChecks++; if (orig.getNumber() != valid.getNumber()) { String output = @@ -147,7 +147,7 @@ public static ArrayList valid(BlockHeader orig, BlockHeader valid) { outputSummary.add(output); } - + if (vStats!=null) vStats.blockChecks++; if (!new BigInteger(1, orig.getGasLimit()).equals(new BigInteger(1, valid.getGasLimit()))) { String output = @@ -158,7 +158,7 @@ public static ArrayList valid(BlockHeader orig, BlockHeader valid) { outputSummary.add(output); } - + if (vStats!=null) vStats.blockChecks++; if (orig.getGasUsed() != valid.getGasUsed()) { String output = @@ -169,7 +169,7 @@ public static ArrayList valid(BlockHeader orig, BlockHeader valid) { outputSummary.add(output); } - + if (vStats!=null) vStats.blockChecks++; if (!toHexString(orig.getExtraData()) .equals(toHexString(valid.getExtraData()))) { diff --git a/rskj-core/src/test/java/org/ethereum/jsontestsuite/validators/LogsValidator.java b/rskj-core/src/test/java/org/ethereum/jsontestsuite/validators/LogsValidator.java index dec75c5a98f..2355da20d5c 100644 --- a/rskj-core/src/test/java/org/ethereum/jsontestsuite/validators/LogsValidator.java +++ b/rskj-core/src/test/java/org/ethereum/jsontestsuite/validators/LogsValidator.java @@ -28,13 +28,13 @@ public class LogsValidator { - public static List valid(List origLogs, List postLogs) { + public static List valid(List origLogs, List postLogs,ValidationStats vStats) { List results = new ArrayList<>(); int i = 0; for (LogInfo postLog : postLogs) { - + if (vStats!=null) vStats.logChecks++; if (origLogs == null || origLogs.size() - 1 < i){ String formattedString = String.format("Log: %s: was expected but doesn't exist: address: %s", i, Hex.toHexString(postLog.getAddress())); @@ -47,7 +47,7 @@ public static List valid(List origLogs, List postLogs) String postAddress = Hex.toHexString(postLog.getAddress()); String realAddress = Hex.toHexString(realLog.getAddress()); - + if (vStats!=null) vStats.logChecks++; if (!postAddress.equals(realAddress)) { String formattedString = String.format("Log: %s: has unexpected address, expected address: %s found address: %s", @@ -57,7 +57,7 @@ public static List valid(List origLogs, List postLogs) String postData = Hex.toHexString(postLog.getData()); String realData = Hex.toHexString(realLog.getData()); - + if (vStats!=null) vStats.logChecks++; if (!postData.equals(realData)) { String formattedString = String.format("Log: %s: has unexpected data, expected data: %s found data: %s", @@ -67,7 +67,7 @@ public static List valid(List origLogs, List postLogs) String postBloom = Hex.toHexString(postLog.getBloom().getData()); String realBloom = Hex.toHexString(realLog.getBloom().getData()); - + if (vStats!=null) vStats.logChecks++; if (!postData.equals(realData)) { String formattedString = String.format("Log: %s: has unexpected bloom, expected bloom: %s found bloom: %s", @@ -82,7 +82,7 @@ public static List valid(List origLogs, List postLogs) for (DataWord postTopic : postTopics) { DataWord realTopic = realTopics.get(j); - + if (vStats!=null) vStats.logChecks++; if (!postTopic.equals(realTopic)) { String formattedString = String.format("Log: %s: has unexpected topic: %s, expected topic: %s found topic: %s", diff --git a/rskj-core/src/test/java/org/ethereum/jsontestsuite/validators/OutputValidator.java b/rskj-core/src/test/java/org/ethereum/jsontestsuite/validators/OutputValidator.java index 9f46285ba92..7922ae8c2d1 100644 --- a/rskj-core/src/test/java/org/ethereum/jsontestsuite/validators/OutputValidator.java +++ b/rskj-core/src/test/java/org/ethereum/jsontestsuite/validators/OutputValidator.java @@ -28,12 +28,12 @@ public class OutputValidator { - public static List valid(String origOutput, String postOutput){ + public static List valid(String origOutput, String postOutput,ValidationStats vStats){ List results = new ArrayList<>(); String postOutputFormated = Hex.toHexString(parseData(postOutput)); - + if (vStats!=null) vStats.outputChecks++; if (!origOutput.equals(postOutputFormated)){ String formattedString = String.format("HReturn: wrong expected: %s, current: %s", postOutputFormated, origOutput); diff --git a/rskj-core/src/test/java/org/ethereum/jsontestsuite/validators/RepositoryValidator.java b/rskj-core/src/test/java/org/ethereum/jsontestsuite/validators/RepositoryValidator.java index c93aeab6a9f..5de5e82af86 100644 --- a/rskj-core/src/test/java/org/ethereum/jsontestsuite/validators/RepositoryValidator.java +++ b/rskj-core/src/test/java/org/ethereum/jsontestsuite/validators/RepositoryValidator.java @@ -30,13 +30,15 @@ public class RepositoryValidator { - public static List valid(Repository currentRepository, Repository postRepository,boolean validateRootHash) { + public static List valid(Repository currentRepository, Repository postRepository, + boolean validateRootHash,boolean validateBalances,ValidationStats vStats) { List results = new ArrayList<>(); Set currentKeys = currentRepository.getAccountsKeys(); Set expectedKeys = postRepository.getAccountsKeys(); + if (vStats!=null) vStats.accountChecks++; if (expectedKeys.size() != currentKeys.size()) { String out = @@ -46,27 +48,30 @@ public static List valid(Repository currentRepository, Repository postRe } for (RskAddress addr : currentKeys) { - List accountResult = AccountValidator.valid(addr, currentRepository, postRepository); + List accountResult = AccountValidator.valid(addr, currentRepository, postRepository,validateBalances,vStats); results.addAll(accountResult); } - + if (vStats!=null) vStats.accountChecks++; Set expectedButAbsent = ByteUtil.difference(expectedKeys, currentKeys); for (RskAddress addr : expectedButAbsent){ String formattedString = String.format("Account: %s: expected but doesn't exist", addr); results.add(formattedString); } - // Compare roots - String postRoot = Hex.toHexString(postRepository.getRoot()); - String currRoot = Hex.toHexString(currentRepository.getRoot()); - if (validateRootHash && !postRoot.equals(currRoot)) { - String formattedString = String.format("Root hash doesn't match: expected: %s current: %s", - postRoot, currRoot); - results.add(formattedString); - } + if (validateRootHash) { + // Compare roots + String postRoot = Hex.toHexString(postRepository.getRoot()); + String currRoot = Hex.toHexString(currentRepository.getRoot()); + if (vStats!=null) vStats.stateChecks++; + if (!postRoot.equals(currRoot)) { + String formattedString = String.format("Root hash doesn't match: expected: %s current: %s", + postRoot, currRoot); + results.add(formattedString); + } + } return results; } diff --git a/rskj-core/src/test/java/org/ethereum/jsontestsuite/validators/ValidationStats.java b/rskj-core/src/test/java/org/ethereum/jsontestsuite/validators/ValidationStats.java new file mode 100644 index 00000000000..abb4c1f07c2 --- /dev/null +++ b/rskj-core/src/test/java/org/ethereum/jsontestsuite/validators/ValidationStats.java @@ -0,0 +1,15 @@ +package org.ethereum.jsontestsuite.validators; + +/** + * Created by Sergio Lerner on 4/24/2019. + */ +public class ValidationStats { + public int accountChecks; + public int storageChecks; + public int balancetChecks; + public int blockChecks; + public int logChecks; + public int outputChecks; + public int stateChecks; + public int skippedChecks; +} diff --git a/rskj-core/src/test/resources/json/StateTests/stSystemOperationsTest.json b/rskj-core/src/test/resources/json/StateTests/stSystemOperationsTest.json index a317ee0941e..c6fc5d48bea 100644 --- a/rskj-core/src/test/resources/json/StateTests/stSystemOperationsTest.json +++ b/rskj-core/src/test/resources/json/StateTests/stSystemOperationsTest.json @@ -7743,7 +7743,7 @@ "createWithInvalidOpcode" : { "env" : { "currentCoinbase" : "2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", - "currentDifficulty" : "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "currentDifficulty" : "0x1fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "currentGasLimit" : "0x0f4240", "currentNumber" : "0x259518", "currentTimestamp" : "0x02", @@ -8632,7 +8632,7 @@ "testRandomTest" : { "env" : { "currentCoinbase" : "2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", - "currentDifficulty" : "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "currentDifficulty" : "0x1fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "currentGasLimit" : "0x0f4240", "currentNumber" : "0x259518", "currentTimestamp" : "0x02",