From 6921f42e31e71a14a94cc72bc4d642fc7b13dd0f Mon Sep 17 00:00:00 2001 From: furszy Date: Sat, 5 Jun 2021 12:16:58 -0300 Subject: [PATCH 01/58] streams: backport OverrideStream class --- src/streams.h | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/streams.h b/src/streams.h index c2ef214bc0b34..6dd5d96012b6e 100644 --- a/src/streams.h +++ b/src/streams.h @@ -23,6 +23,34 @@ #include #include +template +class OverrideStream +{ + Stream* stream; + + const int nType; + const int nVersion; + +public: + OverrideStream(Stream* stream_, int nType_, int nVersion_) : stream(stream_), nType(nType_), nVersion(nVersion_) {} + + template + OverrideStream& operator<<(const T& obj) + { + // Serialize to this stream + ::Serialize(*this->stream, obj, nType, nVersion); + return (*this); + } + + template + OverrideStream& operator>>(T&& obj) + { + // Unserialize from this stream + ::Unserialize(*this->stream, obj, nType, nVersion); + return (*this); + } +}; + /** Double ended buffer combining vector and stream-like interfaces. * * >> and << read and write unformatted data using the above serialization templates. From e2d776a3e63508f084dec3279ed31384ce51e32a Mon Sep 17 00:00:00 2001 From: Larry Ruane Date: Tue, 10 Sep 2019 07:53:09 -0600 Subject: [PATCH 02/58] util: CBufferedFile fixes --- src/streams.h | 35 +++--- src/test/streams_tests.cpp | 240 +++++++++++++++++++++++++++++++++++++ 2 files changed, 258 insertions(+), 17 deletions(-) diff --git a/src/streams.h b/src/streams.h index 6dd5d96012b6e..29f8baeea5f84 100644 --- a/src/streams.h +++ b/src/streams.h @@ -572,19 +572,20 @@ class CBufferedFile readNow = nAvail; if (readNow == 0) return false; - size_t read = fread((void*)&vchBuf[pos], 1, readNow, src); - if (read == 0) { - throw std::ios_base::failure(feof(src) ? "CBufferedFile::Fill : end of file" : "CBufferedFile::Fill : fread failed"); - } else { - nSrcPos += read; - return true; + size_t nBytes = fread((void*)&vchBuf[pos], 1, readNow, src); + if (nBytes == 0) { + throw std::ios_base::failure(feof(src) ? "CBufferedFile::Fill: end of file" : "CBufferedFile::Fill: fread failed"); } + nSrcPos += nBytes; + return true; } public: CBufferedFile(FILE* fileIn, uint64_t nBufSize, uint64_t nRewindIn, int nTypeIn, int nVersionIn) : nType(nTypeIn), nVersion(nVersionIn), nSrcPos(0), nReadPos(0), nReadLimit((uint64_t)(-1)), nRewind(nRewindIn), vchBuf(nBufSize, 0) { + if (nRewindIn >= nBufSize) + throw std::ios_base::failure("Rewind limit must be less than buffer size"); src = fileIn; } @@ -616,8 +617,6 @@ class CBufferedFile { if (nSize + nReadPos > nReadLimit) throw std::ios_base::failure("Read attempted past buffer limit"); - if (nSize + nRewind > vchBuf.size()) - throw std::ios_base::failure("Read larger than buffer size"); while (nSize > 0) { if (nReadPos == nSrcPos) Fill(); @@ -640,19 +639,21 @@ class CBufferedFile return nReadPos; } - // rewind to a given reading position - bool SetPos(uint64_t nPos) - { - nReadPos = nPos; - if (nReadPos + nRewind < nSrcPos) { - nReadPos = nSrcPos - nRewind; + //! rewind to a given reading position + bool SetPos(uint64_t nPos) { + size_t bufsize = vchBuf.size(); + if (nPos + bufsize < nSrcPos) { + // rewinding too far, rewind as far as possible + nReadPos = nSrcPos - bufsize; return false; - } else if (nReadPos > nSrcPos) { + } + if (nPos > nSrcPos) { + // can't go this far forward, go as far as possible nReadPos = nSrcPos; return false; - } else { - return true; } + nReadPos = nPos; + return true; } bool Seek(uint64_t nPos) diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index ea0fbc7a67229..6b9b07fd6e660 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "random.h" #include "streams.h" #include "test/test_pivx.h" @@ -67,4 +68,243 @@ BOOST_AUTO_TEST_CASE(streams_vector_writer) vch.clear(); } +BOOST_AUTO_TEST_CASE(streams_buffered_file) +{ + FILE* file = fsbridge::fopen("streams_test_tmp", "w+b"); + // The value at each offset is the offset. + for (uint8_t j = 0; j < 40; ++j) { + fwrite(&j, 1, 1, file); + } + rewind(file); + + // The buffer size (second arg) must be greater than the rewind + // amount (third arg). + try { + CBufferedFile bfbad(file, 25, 25, 222, 333); + BOOST_CHECK(false); + } catch (const std::exception& e) { + BOOST_CHECK(strstr(e.what(), + "Rewind limit must be less than buffer size") != nullptr); + } + + // The buffer is 25 bytes, allow rewinding 10 bytes. + CBufferedFile bf(file, 25, 10, 222, 333); + BOOST_CHECK(!bf.eof()); + + uint8_t i; + bf >> i; + BOOST_CHECK_EQUAL(i, 0); + bf >> i; + BOOST_CHECK_EQUAL(i, 1); + + // After reading bytes 0 and 1, we're positioned at 2. + BOOST_CHECK_EQUAL(bf.GetPos(), 2); + + // Rewind to offset 0, ok (within the 10 byte window). + BOOST_CHECK(bf.SetPos(0)); + bf >> i; + BOOST_CHECK_EQUAL(i, 0); + + // We can go forward to where we've been, but beyond may fail. + BOOST_CHECK(bf.SetPos(2)); + bf >> i; + BOOST_CHECK_EQUAL(i, 2); + + // If you know the maximum number of bytes that should be + // read to deserialize the variable, you can limit the read + // extent. The current file offset is 3, so the following + // SetLimit() allows zero bytes to be read. + BOOST_CHECK(bf.SetLimit(3)); + try { + bf >> i; + BOOST_CHECK(false); + } catch (const std::exception& e) { + BOOST_CHECK(strstr(e.what(), + "Read attempted past buffer limit") != nullptr); + } + // The default argument removes the limit completely. + BOOST_CHECK(bf.SetLimit()); + // The read position should still be at 3 (no change). + BOOST_CHECK_EQUAL(bf.GetPos(), 3); + + // Read from current offset, 3, forward until position 10. + for (uint8_t j = 3; j < 10; ++j) { + bf >> i; + BOOST_CHECK_EQUAL(i, j); + } + BOOST_CHECK_EQUAL(bf.GetPos(), 10); + + // We're guaranteed (just barely) to be able to rewind to zero. + BOOST_CHECK(bf.SetPos(0)); + BOOST_CHECK_EQUAL(bf.GetPos(), 0); + bf >> i; + BOOST_CHECK_EQUAL(i, 0); + + // We can set the position forward again up to the farthest + // into the stream we've been, but no farther. (Attempting + // to go farther may succeed, but it's not guaranteed.) + BOOST_CHECK(bf.SetPos(10)); + bf >> i; + BOOST_CHECK_EQUAL(i, 10); + BOOST_CHECK_EQUAL(bf.GetPos(), 11); + + // Now it's only guaranteed that we can rewind to offset 1 + // (current read position, 11, minus rewind amount, 10). + BOOST_CHECK(bf.SetPos(1)); + BOOST_CHECK_EQUAL(bf.GetPos(), 1); + bf >> i; + BOOST_CHECK_EQUAL(i, 1); + + // We can stream into large variables, even larger than + // the buffer size. + BOOST_CHECK(bf.SetPos(11)); + { + uint8_t a[40 - 11]; + bf >> a; + for (uint8_t j = 0; j < sizeof(a); ++j) { + BOOST_CHECK_EQUAL(a[j], 11 + j); + } + } + BOOST_CHECK_EQUAL(bf.GetPos(), 40); + + // We've read the entire file, the next read should throw. + try { + bf >> i; + BOOST_CHECK(false); + } catch (const std::exception& e) { + BOOST_CHECK(strstr(e.what(), + "CBufferedFile::Fill: end of file") != nullptr); + } + // Attempting to read beyond the end sets the EOF indicator. + BOOST_CHECK(bf.eof()); + + // Still at offset 40, we can go back 10, to 30. + BOOST_CHECK_EQUAL(bf.GetPos(), 40); + BOOST_CHECK(bf.SetPos(30)); + bf >> i; + BOOST_CHECK_EQUAL(i, 30); + BOOST_CHECK_EQUAL(bf.GetPos(), 31); + + // We're too far to rewind to position zero. + BOOST_CHECK(!bf.SetPos(0)); + // But we should now be positioned at least as far back as allowed + // by the rewind window (relative to our farthest read position, 40). + BOOST_CHECK(bf.GetPos() <= 30); + + // We can explicitly close the file, or the destructor will do it. + bf.fclose(); + + fs::remove("streams_test_tmp"); +} + +BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) +{ + // Make this test deterministic. + SeedInsecureRand(true); + + for (int rep = 0; rep < 50; ++rep) { + FILE* file = fsbridge::fopen("streams_test_tmp", "w+b"); + size_t fileSize = InsecureRandRange(256); + for (uint8_t i = 0; i < fileSize; ++i) { + fwrite(&i, 1, 1, file); + } + rewind(file); + + size_t bufSize = InsecureRandRange(300) + 1; + size_t rewindSize = InsecureRandRange(bufSize); + CBufferedFile bf(file, bufSize, rewindSize, 222, 333); + size_t currentPos = 0; + size_t maxPos = 0; + for (int step = 0; step < 100; ++step) { + if (currentPos >= fileSize) + break; + + // We haven't read to the end of the file yet. + BOOST_CHECK(!bf.eof()); + BOOST_CHECK_EQUAL(bf.GetPos(), currentPos); + + // Pretend the file consists of a series of objects of varying + // sizes; the boundaries of the objects can interact arbitrarily + // with the CBufferFile's internal buffer. These first three + // cases simulate objects of various sizes (1, 2, 5 bytes). + switch (InsecureRandRange(5)) { + case 0: { + uint8_t a[1]; + if (currentPos + 1 > fileSize) + continue; + bf.SetLimit(currentPos + 1); + bf >> a; + for (uint8_t i = 0; i < 1; ++i) { + BOOST_CHECK_EQUAL(a[i], currentPos); + currentPos++; + } + break; + } + case 1: { + uint8_t a[2]; + if (currentPos + 2 > fileSize) + continue; + bf.SetLimit(currentPos + 2); + bf >> a; + for (uint8_t i = 0; i < 2; ++i) { + BOOST_CHECK_EQUAL(a[i], currentPos); + currentPos++; + } + break; + } + case 2: { + uint8_t a[5]; + if (currentPos + 5 > fileSize) + continue; + bf.SetLimit(currentPos + 5); + bf >> a; + for (uint8_t i = 0; i < 5; ++i) { + BOOST_CHECK_EQUAL(a[i], currentPos); + currentPos++; + } + break; + } + case 3: { + // Find a byte value (that is at or ahead of the current position). + size_t find = currentPos + InsecureRandRange(8); + if (find >= fileSize) + find = fileSize - 1; + bf.FindByte(static_cast(find)); + // The value at each offset is the offset. + BOOST_CHECK_EQUAL(bf.GetPos(), find); + currentPos = find; + + bf.SetLimit(currentPos + 1); + uint8_t i; + bf >> i; + BOOST_CHECK_EQUAL(i, currentPos); + currentPos++; + break; + } + case 4: { + size_t requestPos = InsecureRandRange(maxPos + 4); + bool okay = bf.SetPos(requestPos); + // The new position may differ from the requested position + // because we may not be able to rewind beyond the rewind + // window, and we may not be able to move forward beyond the + // farthest position we've reached so far. + currentPos = bf.GetPos(); + BOOST_CHECK_EQUAL(okay, currentPos == requestPos); + // Check that we can position within the rewind window. + if (requestPos <= maxPos && + maxPos > rewindSize && + requestPos >= maxPos - rewindSize) { + // We requested a position within the rewind window. + BOOST_CHECK(okay); + } + break; + } + } + if (maxPos < currentPos) + maxPos = currentPos; + } + } + fs::remove("streams_test_tmp"); +} + BOOST_AUTO_TEST_SUITE_END() From 082baa311a456e377d0bb7506e218e13019b6752 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Sun, 26 Jul 2020 22:28:56 +0300 Subject: [PATCH 03/58] refactor: Drop unused CBufferedFile::Seek() --- src/streams.h | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/src/streams.h b/src/streams.h index 29f8baeea5f84..129a6c9a398ab 100644 --- a/src/streams.h +++ b/src/streams.h @@ -656,23 +656,9 @@ class CBufferedFile return true; } - bool Seek(uint64_t nPos) - { - long nLongPos = nPos; - if (nPos != (uint64_t)nLongPos) - return false; - if (fseek(src, nLongPos, SEEK_SET)) - return false; - nLongPos = ftell(src); - nSrcPos = nLongPos; - nReadPos = nLongPos; - return true; - } - - // prevent reading beyond a certain position - // no argument removes the limit - bool SetLimit(uint64_t nPos = (uint64_t)(-1)) - { + //! prevent reading beyond a certain position + //! no argument removes the limit + bool SetLimit(uint64_t nPos = std::numeric_limits::max()) { if (nPos < nReadPos) return false; nReadLimit = nPos; From 3660487a6e540605519555a0cc22994a6f2677b9 Mon Sep 17 00:00:00 2001 From: Marco Date: Thu, 3 Jun 2021 11:41:04 -0300 Subject: [PATCH 04/58] net: Use C++11 member initialization in protocol --- src/protocol.cpp | 18 ------------------ src/protocol.h | 17 +++++++---------- 2 files changed, 7 insertions(+), 28 deletions(-) diff --git a/src/protocol.cpp b/src/protocol.cpp index 4a4379618f5d5..81e353c849a60 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -153,24 +153,6 @@ bool CMessageHeader::IsValid(const MessageStartChars& pchMessageStartIn) const return true; } - -CAddress::CAddress() : CService() -{ - Init(); -} - -CAddress::CAddress(CService ipIn, ServiceFlags nServicesIn) : CService(ipIn) -{ - Init(); - nServices = nServicesIn; -} - -void CAddress::Init() -{ - nServices = NODE_NONE; - nTime = 100000000; -} - CInv::CInv() { type = 0; diff --git a/src/protocol.h b/src/protocol.h index fce762bf6d75e..3a49c1f69f658 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -297,15 +297,15 @@ enum ServiceFlags : uint64_t { /** A CService with information about it as peer */ class CAddress : public CService { -public: - CAddress(); - explicit CAddress(CService ipIn, ServiceFlags nServicesIn); + static constexpr uint32_t TIME_INIT{100000000}; - void Init(); +public: + CAddress() : CService{} {}; + explicit CAddress(CService ipIn, ServiceFlags nServicesIn) : CService{ipIn}, nServices{nServicesIn} {}; SERIALIZE_METHODS(CAddress, obj) { - SER_READ(obj, obj.Init()); + SER_READ(obj, obj.nTime = TIME_INIT); int nVersion = s.GetVersion(); if (s.GetType() & SER_DISK) { READWRITE(nVersion); @@ -318,12 +318,9 @@ class CAddress : public CService READWRITEAS(CService, obj); } - // TODO: make private (improves encapsulation) -public: - ServiceFlags nServices; - + ServiceFlags nServices{NODE_NONE}; // disk and network only - unsigned int nTime; + uint32_t nTime{TIME_INIT}; }; /** getdata message types */ From c3c04e4fd1103bd3f6a6935d7ce4f580e1207de8 Mon Sep 17 00:00:00 2001 From: furszy Date: Sat, 5 Jun 2021 10:45:08 -0300 Subject: [PATCH 05/58] net: Better misbehaving logging Adaptation of btc@d3a185a33b7bc09e4ca998f42f1f9aea8177ef8a This moves the error messages for misbehavior (when available) into the line that reports the misbehavior, as well as moves the logging to the `net` category. This is a continuation of btc#11583 and avoids serious-looking errors due to misbehaving peers. To do this, Misbehaving() gains an optional `message` argument. E.g. change: 2018-01-18 16:02:27 Misbehaving: x.x.x.x:62174 peer=164603 (80 -> 100) BAN THRESHOLD EXCEEDED 2018-01-18 16:02:27 ERROR: non-continuous headers sequence to 2018-01-18 16:02:27 Misbehaving: x.x.x.x:62174 peer=164603 (80 -> 100) BAN THRESHOLD EXCEEDED: non-continuous headers sequence --- src/net_processing.cpp | 54 +++++++++++++++++++++++------------------- src/net_processing.h | 2 +- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 0dc3063820bee..73d2fad66684a 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -583,22 +583,24 @@ unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) } // Requires cs_main. -void Misbehaving(NodeId pnode, int howmuch) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +void Misbehaving(NodeId pnode, int howmuch, const std::string& message) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { if (howmuch == 0) return; CNodeState* state = State(pnode); - if (state == NULL) + if (state == nullptr) return; state->nMisbehavior += howmuch; int banscore = gArgs.GetArg("-banscore", DEFAULT_BANSCORE_THRESHOLD); + std::string message_prefixed = message.empty() ? "" : (": " + message); if (state->nMisbehavior >= banscore && state->nMisbehavior - howmuch < banscore) { - LogPrintf("Misbehaving: %s (%d -> %d) BAN THRESHOLD EXCEEDED\n", state->name, state->nMisbehavior - howmuch, state->nMisbehavior); + LogPrint(BCLog::NET, "%s: %s peer=%d (%d -> %d) BAN THRESHOLD EXCEEDED%s\n", __func__, state->name, pnode, state->nMisbehavior-howmuch, state->nMisbehavior, message_prefixed); state->fShouldBan = true; - } else - LogPrintf("Misbehaving: %s (%d -> %d)\n", state->name, state->nMisbehavior - howmuch, state->nMisbehavior); + } else { + LogPrint(BCLog::NET, "%s: %s peer=%d (%d -> %d)%s\n", __func__, state->name, pnode, state->nMisbehavior-howmuch, state->nMisbehavior, message_prefixed); + } } static void CheckBlockSpam(NodeId nodeId, const uint256& hashBlock) @@ -1277,8 +1279,8 @@ bool static ProcessMessage(CNode* pfrom, std::string strCommand, CDataStream& vR return true; if (vAddr.size() > MAX_ADDR_TO_SEND) { LOCK(cs_main); - Misbehaving(pfrom->GetId(), 20); - return error("message addr size() = %u", vAddr.size()); + Misbehaving(pfrom->GetId(), 20, strprintf("message addr size() = %u", vAddr.size())); + return false; } // Store the new addresses @@ -1317,8 +1319,8 @@ bool static ProcessMessage(CNode* pfrom, std::string strCommand, CDataStream& vR vRecv >> vInv; if (vInv.size() > MAX_INV_SZ) { LOCK(cs_main); - Misbehaving(pfrom->GetId(), 20); - return error("peer=%d message inv size() = %u", pfrom->GetId(), vInv.size()); + Misbehaving(pfrom->GetId(), 20, strprintf("message inv size() = %u", vInv.size())); + return false; } LOCK(cs_main); @@ -1333,8 +1335,8 @@ bool static ProcessMessage(CNode* pfrom, std::string strCommand, CDataStream& vR // Reject deprecated messages if (inv.type == MSG_TXLOCK_REQUEST || inv.type == MSG_TXLOCK_VOTE) { - Misbehaving(pfrom->GetId(), 20); - return error("message inv deprecated %d", (int)inv.type); + Misbehaving(pfrom->GetId(), 100, strprintf("message inv deprecated %d", (int)inv.type)); + return false; } pfrom->AddInventoryKnown(inv); @@ -1367,8 +1369,8 @@ bool static ProcessMessage(CNode* pfrom, std::string strCommand, CDataStream& vR vRecv >> vInv; if (vInv.size() > MAX_INV_SZ) { LOCK(cs_main); - Misbehaving(pfrom->GetId(), 20); - return error("peer=%d message getdata size() = %u", pfrom->GetId(), vInv.size()); + Misbehaving(pfrom->GetId(), 20, strprintf("message getdata size() = %u", vInv.size())); + return false; } if (vInv.size() != 1) @@ -1483,8 +1485,8 @@ bool static ProcessMessage(CNode* pfrom, std::string strCommand, CDataStream& vR if (ptx->ContainsZerocoins()) { // Don't even try to check zerocoins at all. - Misbehaving(pfrom->GetId(), 100); - LogPrint(BCLog::NET, " misbehaving peer, received a zc transaction, peer: %s\n", pfrom->GetAddrName()); + Misbehaving(pfrom->GetId(), 100, strprintf("received a zc transaction")); + return false; } if (AcceptToMemoryPool(mempool, state, ptx, true, &fMissingInputs, false, ignoreFees)) { @@ -1622,8 +1624,8 @@ bool static ProcessMessage(CNode* pfrom, std::string strCommand, CDataStream& vR unsigned int nCount = ReadCompactSize(vRecv); if (nCount > MAX_HEADERS_RESULTS) { LOCK(cs_main); - Misbehaving(pfrom->GetId(), 20); - return error("headers message size = %u", nCount); + Misbehaving(pfrom->GetId(), 20, strprintf("headers message size = %u", nCount)); + return false; } headers.resize(nCount); for (unsigned int n = 0; n < nCount; n++) { @@ -1641,8 +1643,8 @@ bool static ProcessMessage(CNode* pfrom, std::string strCommand, CDataStream& vR for (const CBlockHeader& header : headers) { CValidationState state; if (pindexLast != NULL && header.hashPrevBlock != pindexLast->GetBlockHash()) { - Misbehaving(pfrom->GetId(), 20); - return error("non-continuous headers sequence"); + Misbehaving(pfrom->GetId(), 20, "non-continuous headers sequence"); + return false; } /*TODO: this has a CBlock cast on it so that it will compile. There should be a solution for this @@ -1651,10 +1653,12 @@ bool static ProcessMessage(CNode* pfrom, std::string strCommand, CDataStream& vR if (!AcceptBlockHeader((CBlock)header, state, &pindexLast)) { int nDoS; if (state.IsInvalid(nDoS)) { - if (nDoS > 0) - Misbehaving(pfrom->GetId(), nDoS); - std::string strError = "invalid header received " + header.GetHash().ToString(); - return error(strError.c_str()); + if (nDoS > 0) { + Misbehaving(pfrom->GetId(), nDoS, "invalid header received"); + } else { + LogPrint(BCLog::NET, "peer=%d: invalid header received\n", pfrom->GetId()); + } + return false; } } } @@ -1818,9 +1822,9 @@ bool static ProcessMessage(CNode* pfrom, std::string strCommand, CDataStream& vR (strCommand == NetMsgType::FILTERLOAD || strCommand == NetMsgType::FILTERADD || strCommand == NetMsgType::FILTERCLEAR)) { - LogPrintf("bloom message=%s\n", strCommand); LOCK(cs_main); - Misbehaving(pfrom->GetId(), 100); + Misbehaving(pfrom->GetId(), 100, "banning, filter received."); + return false; } else if (strCommand == NetMsgType::FILTERLOAD) { diff --git a/src/net_processing.h b/src/net_processing.h index 148fa33ee4e37..3ec8bea253416 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -69,6 +69,6 @@ struct CNodeStateStats { /** Get statistics from node state */ bool GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats); /** Increase a node's misbehavior score. */ -void Misbehaving(NodeId nodeid, int howmuch) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +void Misbehaving(NodeId nodeid, int howmuch, const std::string& message="") EXCLUSIVE_LOCKS_REQUIRED(cs_main); #endif // BITCOIN_NET_PROCESSING_H From cb160de89257e3fb0aa044cb3c991b8d864538ea Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Thu, 28 Mar 2019 15:21:31 -0400 Subject: [PATCH 06/58] netaddress: Update CNetAddr for ORCHIDv2 The original ORCHID prefix was deprecated as of 2014-03, the new ORCHIDv2 prefix was allocated by RFC7343 as of 2014-07. We did not consider the original ORCHID prefix routable, and I don't see any reason to consider the new one to be either. --- src/netaddress.cpp | 7 ++++++- src/netaddress.h | 3 ++- src/test/netbase_tests.cpp | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 4d92a76e1bf6e..ebf8859413511 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -174,6 +174,11 @@ bool CNetAddr::IsRFC4843() const return (GetByte(15) == 0x20 && GetByte(14) == 0x01 && GetByte(13) == 0x00 && (GetByte(12) & 0xF0) == 0x10); } +bool CNetAddr::IsRFC7343() const +{ + return (GetByte(15) == 0x20 && GetByte(14) == 0x01 && GetByte(13) == 0x00 && (GetByte(12) & 0xF0) == 0x20); +} + bool CNetAddr::IsHeNet() const { return (GetByte(15) == 0x20 && GetByte(14) == 0x01 && GetByte(13) == 0x04 && GetByte(12) == 0x70); @@ -239,7 +244,7 @@ bool CNetAddr::IsValid() const bool CNetAddr::IsRoutable() const { - return IsValid() && !(IsRFC1918() || IsRFC2544() || IsRFC3927() || IsRFC4862() || IsRFC6598() || IsRFC5737() || (IsRFC4193() && !IsTor()) || IsRFC4843() || IsLocal() || IsInternal()); + return IsValid() && !(IsRFC1918() || IsRFC2544() || IsRFC3927() || IsRFC4862() || IsRFC6598() || IsRFC5737() || (IsRFC4193() && !IsTor()) || IsRFC4843() || IsRFC7343() || IsLocal() || IsInternal()); } bool CNetAddr::IsInternal() const diff --git a/src/netaddress.h b/src/netaddress.h index 5f54c6fa843a8..25f5e641de9ab 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -65,7 +65,8 @@ class CNetAddr bool IsRFC3964() const; // IPv6 6to4 tunnelling (2002::/16) bool IsRFC4193() const; // IPv6 unique local (FC00::/7) bool IsRFC4380() const; // IPv6 Teredo tunnelling (2001::/32) - bool IsRFC4843() const; // IPv6 ORCHID (2001:10::/28) + bool IsRFC4843() const; // IPv6 ORCHID (deprecated) (2001:10::/28) + bool IsRFC7343() const; // IPv6 ORCHIDv2 (2001:20::/28) bool IsRFC4862() const; // IPv6 autoconfig (FE80::/64) bool IsRFC6052() const; // IPv6 well-known prefix (64:FF9B::/96) bool IsRFC6145() const; // IPv6 IPv4-translated address (::FFFF:0:0:0/96) diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index ccd95a1253b65..9f9831c8aabaf 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -60,6 +60,7 @@ BOOST_AUTO_TEST_CASE(netbase_properties) BOOST_CHECK(ResolveIP("FC00::").IsRFC4193()); BOOST_CHECK(ResolveIP("2001::2").IsRFC4380()); BOOST_CHECK(ResolveIP("2001:10::").IsRFC4843()); + BOOST_CHECK(ResolveIP("2001:20::").IsRFC7343()); BOOST_CHECK(ResolveIP("FE80::").IsRFC4862()); BOOST_CHECK(ResolveIP("64:FF9B::").IsRFC6052()); BOOST_CHECK(ResolveIP("FD87:D87E:EB43:edb1:8e4:3588:e546:35ca").IsTor()); From ad57dfc282e8469c773ddd9b17bf053db2f73140 Mon Sep 17 00:00:00 2001 From: Vasil Dimov Date: Thu, 14 May 2020 10:47:46 +0200 Subject: [PATCH 07/58] net: document `enum Network` --- src/netaddress.h | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/netaddress.h b/src/netaddress.h index 25f5e641de9ab..9b2d1dbffd0ba 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -18,13 +18,36 @@ #include #include -enum Network { +/** + * A network type. + * @note An address may belong to more than one network, for example `10.0.0.1` + * belongs to both `NET_UNROUTABLE` and `NET_IPV4`. + * Keep these sequential starting from 0 and `NET_MAX` as the last entry. + * We have loops like `for (int i = 0; i < NET_MAX; i++)` that expect to iterate + * over all enum values and also `GetExtNetwork()` "extends" this enum by + * introducing standalone constants starting from `NET_MAX`. + */ +enum Network +{ + /// Addresses from these networks are not publicly routable on the global Internet. NET_UNROUTABLE = 0, + + /// IPv4 NET_IPV4, + + /// IPv6 NET_IPV6, + + /// TORv2 NET_ONION, + + /// A set of dummy addresses that map a name to an IPv6 address. These + /// addresses belong to RFC4193's fc00::/7 subnet (unique-local addresses). + /// We use them to map a string or FQDN to an IPv6 address in CAddrMan to + /// keep track of which DNS seeds were used. NET_INTERNAL, + /// Dummy value to indicate the number of NET_* constants. NET_MAX, }; From 86d73fb16edb336ad1ed2547d22fe85ea84341e1 Mon Sep 17 00:00:00 2001 From: Vasil Dimov Date: Thu, 14 May 2020 17:03:11 +0200 Subject: [PATCH 08/58] net: save the network type explicitly in CNetAddr Before this change, we would analyze the contents of `CNetAddr::ip[16]` in order to tell which type is an address. Change this by introducing a new member `CNetAddr::m_net` that explicitly tells the type of the address. This is necessary because in BIP155 we will not be able to tell the address type by just looking at its raw representation (e.g. both TORv3 and I2P are "seemingly random" 32 bytes). As a side effect of this change we no longer need to store IPv4 addresses encoded as IPv6 addresses - we can store them in proper 4 bytes (will be done in a separate commit). Also the code gets somewhat simplified - instead of `memcmp(ip, pchIPv4, sizeof(pchIPv4)) == 0` we can use `m_net == NET_IPV4`. Co-authored-by: Carl Dong --- src/netaddress.cpp | 91 ++++++++++++++++++++++---------------- src/netaddress.h | 47 +++++++++++++++++--- src/test/netbase_tests.cpp | 16 ++++++- 3 files changed, 107 insertions(+), 47 deletions(-) diff --git a/src/netaddress.cpp b/src/netaddress.cpp index ebf8859413511..07e1be2473636 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -25,19 +25,35 @@ void CNetAddr::Init() void CNetAddr::SetIP(const CNetAddr& ipIn) { + m_net = ipIn.m_net; memcpy(ip, ipIn.ip, sizeof(ip)); } +void CNetAddr::SetLegacyIPv6(const uint8_t ipv6[16]) +{ + if (memcmp(ipv6, pchIPv4, sizeof(pchIPv4)) == 0) { + m_net = NET_IPV4; + } else if (memcmp(ipv6, pchOnionCat, sizeof(pchOnionCat)) == 0) { + m_net = NET_ONION; + } else if (memcmp(ipv6, g_internal_prefix, sizeof(g_internal_prefix)) == 0) { + m_net = NET_INTERNAL; + } else { + m_net = NET_IPV6; + } + memcpy(ip, ipv6, 16); +} + void CNetAddr::SetRaw(Network network, const uint8_t* ip_in) { switch(network) { case NET_IPV4: + m_net = NET_IPV4; memcpy(ip, pchIPv4, 12); memcpy(ip+12, ip_in, 4); break; case NET_IPV6: - memcpy(ip, ip_in, 16); + SetLegacyIPv6(ip_in); break; default: assert(!"invalid network"); @@ -49,6 +65,7 @@ bool CNetAddr::SetInternal(const std::string &name) if (name.empty()) { return false; } + m_net = NET_INTERNAL; unsigned char hash[32] = {}; CSHA256().Write((const unsigned char*)name.data(), name.size()).Finalize(hash); memcpy(ip, g_internal_prefix, sizeof(g_internal_prefix)); @@ -62,6 +79,7 @@ bool CNetAddr::SetSpecial(const std::string& strName) std::vector vchAddr = DecodeBase32(strName.substr(0, strName.size() - 6).c_str()); if (vchAddr.size() != 16-sizeof(pchOnionCat)) return false; + m_net = NET_ONION; memcpy(ip, pchOnionCat, sizeof(pchOnionCat)); for (unsigned int i=0; i<16-sizeof(pchOnionCat); i++) ip[i + sizeof(pchOnionCat)] = vchAddr[i]; @@ -91,15 +109,9 @@ unsigned int CNetAddr::GetByte(int n) const return ip[15-n]; } -bool CNetAddr::IsIPv4() const -{ - return (memcmp(ip, pchIPv4, sizeof(pchIPv4)) == 0); -} +bool CNetAddr::IsIPv4() const { return m_net == NET_IPV4; } -bool CNetAddr::IsIPv6() const -{ - return (!IsIPv4() && !IsTor() && !IsInternal()); -} +bool CNetAddr::IsIPv6() const { return m_net == NET_IPV6; } bool CNetAddr::IsRFC1918() const { @@ -133,50 +145,54 @@ bool CNetAddr::IsRFC5737() const bool CNetAddr::IsRFC3849() const { - return GetByte(15) == 0x20 && GetByte(14) == 0x01 && GetByte(13) == 0x0D && GetByte(12) == 0xB8; + return IsIPv6() && GetByte(15) == 0x20 && GetByte(14) == 0x01 && + GetByte(13) == 0x0D && GetByte(12) == 0xB8; } bool CNetAddr::IsRFC3964() const { - return (GetByte(15) == 0x20 && GetByte(14) == 0x02); + return IsIPv6() && GetByte(15) == 0x20 && GetByte(14) == 0x02; } bool CNetAddr::IsRFC6052() const { static const unsigned char pchRFC6052[] = {0,0x64,0xFF,0x9B,0,0,0,0,0,0,0,0}; - return (memcmp(ip, pchRFC6052, sizeof(pchRFC6052)) == 0); + return IsIPv6() && memcmp(ip, pchRFC6052, sizeof(pchRFC6052)) == 0; } bool CNetAddr::IsRFC4380() const { - return (GetByte(15) == 0x20 && GetByte(14) == 0x01 && GetByte(13) == 0 && GetByte(12) == 0); + return IsIPv6() && GetByte(15) == 0x20 && GetByte(14) == 0x01 && GetByte(13) == 0 && + GetByte(12) == 0; } bool CNetAddr::IsRFC4862() const { static const unsigned char pchRFC4862[] = {0xFE,0x80,0,0,0,0,0,0}; - return (memcmp(ip, pchRFC4862, sizeof(pchRFC4862)) == 0); + return IsIPv6() && memcmp(ip, pchRFC4862, sizeof(pchRFC4862)) == 0; } bool CNetAddr::IsRFC4193() const { - return ((GetByte(15) & 0xFE) == 0xFC); + return IsIPv6() && (GetByte(15) & 0xFE) == 0xFC; } bool CNetAddr::IsRFC6145() const { static const unsigned char pchRFC6145[] = {0,0,0,0,0,0,0,0,0xFF,0xFF,0,0}; - return (memcmp(ip, pchRFC6145, sizeof(pchRFC6145)) == 0); + return IsIPv6() && memcmp(ip, pchRFC6145, sizeof(pchRFC6145)) == 0; } bool CNetAddr::IsRFC4843() const { - return (GetByte(15) == 0x20 && GetByte(14) == 0x01 && GetByte(13) == 0x00 && (GetByte(12) & 0xF0) == 0x10); + return IsIPv6() && GetByte(15) == 0x20 && GetByte(14) == 0x01 && + GetByte(13) == 0x00 && (GetByte(12) & 0xF0) == 0x10; } bool CNetAddr::IsRFC7343() const { - return (GetByte(15) == 0x20 && GetByte(14) == 0x01 && GetByte(13) == 0x00 && (GetByte(12) & 0xF0) == 0x20); + return IsIPv6() && GetByte(15) == 0x20 && GetByte(14) == 0x01 && + GetByte(13) == 0x00 && (GetByte(12) & 0xF0) == 0x20; } bool CNetAddr::IsHeNet() const @@ -184,10 +200,13 @@ bool CNetAddr::IsHeNet() const return (GetByte(15) == 0x20 && GetByte(14) == 0x01 && GetByte(13) == 0x04 && GetByte(12) == 0x70); } -bool CNetAddr::IsTor() const -{ - return (memcmp(ip, pchOnionCat, sizeof(pchOnionCat)) == 0); -} +/** + * @returns Whether or not this is a dummy address that maps an onion address + * into IPv6. + * + * @see CNetAddr::SetSpecial(const std::string &) + */ +bool CNetAddr::IsTor() const { return m_net == NET_ONION; } bool CNetAddr::IsLocal() const { @@ -195,10 +214,10 @@ bool CNetAddr::IsLocal() const if (IsIPv4() && (GetByte(3) == 127 || GetByte(3) == 0)) return true; - // IPv6 loopback (::1/128) - static const unsigned char pchLocal[16] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1}; - if (memcmp(ip, pchLocal, 16) == 0) - return true; + // IPv6 loopback (::1/128) + static const unsigned char pchLocal[16] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1}; + if (IsIPv6() && memcmp(ip, pchLocal, 16) == 0) + return true; return false; } @@ -211,12 +230,12 @@ bool CNetAddr::IsValid() const // header20 vectorlen3 addr26 addr26 addr26 header20 vectorlen3 addr26 addr26 addr26... // so if the first length field is garbled, it reads the second batch // of addr misaligned by 3 bytes. - if (memcmp(ip, pchIPv4+3, sizeof(pchIPv4)-3) == 0) + if (IsIPv6() && memcmp(ip, pchIPv4+3, sizeof(pchIPv4)-3) == 0) return false; // unspecified IPv6 address (::/128) unsigned char ipNone6[16] = {}; - if (memcmp(ip, ipNone6, 16) == 0) + if (IsIPv6() && memcmp(ip, ipNone6, 16) == 0) return false; // documentation IPv6 address @@ -249,7 +268,7 @@ bool CNetAddr::IsRoutable() const bool CNetAddr::IsInternal() const { - return memcmp(ip, g_internal_prefix, sizeof(g_internal_prefix)) == 0; + return m_net == NET_INTERNAL; } enum Network CNetAddr::GetNetwork() const @@ -260,13 +279,7 @@ enum Network CNetAddr::GetNetwork() const if (!IsRoutable()) return NET_UNROUTABLE; - if (IsIPv4()) - return NET_IPV4; - - if (IsTor()) - return NET_ONION; - - return NET_IPV6; + return m_net; } std::string CNetAddr::ToStringIP() const @@ -300,7 +313,7 @@ std::string CNetAddr::ToString() const bool operator==(const CNetAddr& a, const CNetAddr& b) { - return (memcmp(a.ip, b.ip, 16) == 0); + return a.m_net == b.m_net && memcmp(a.ip, b.ip, 16) == 0; } bool operator!=(const CNetAddr& a, const CNetAddr& b) @@ -310,7 +323,7 @@ bool operator!=(const CNetAddr& a, const CNetAddr& b) bool operator<(const CNetAddr& a, const CNetAddr& b) { - return (memcmp(a.ip, b.ip, 16) < 0); + return a.m_net < b.m_net || (a.m_net == b.m_net && memcmp(a.ip, b.ip, 16) < 0); } bool CNetAddr::GetInAddr(struct in_addr* pipv4Addr) const @@ -726,7 +739,7 @@ CSubNet::CSubNet(const CNetAddr &addr): bool CSubNet::Match(const CNetAddr &addr) const { - if (!valid || !addr.IsValid()) + if (!valid || !addr.IsValid() || network.m_net != addr.m_net) return false; for(int x=0; x<16; ++x) if ((addr.ip[x] & netmask[x]) != network.ip[x]) diff --git a/src/netaddress.h b/src/netaddress.h index 9b2d1dbffd0ba..a24c21577b017 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -51,12 +51,19 @@ enum Network NET_MAX, }; -/** IP address (IPv6, or IPv4 using mapped IPv6 range (::FFFF:0:0/96)) */ +/** + * Network address. + */ class CNetAddr { protected: + /** + * Network to which this address belongs. + */ + Network m_net{NET_IPV6}; + unsigned char ip[16]; // in network byte order - uint32_t scopeId{0}; // for scoped/link-local ipv6 addresses + uint32_t scopeId{0}; // for scoped/link-local ipv6 addresses public: CNetAddr(); @@ -65,9 +72,17 @@ class CNetAddr void SetIP(const CNetAddr& ip); /** - * Set raw IPv4 or IPv6 address (in network byte order) - * @note Only NET_IPV4 and NET_IPV6 are allowed for network. - */ + * Set from a legacy IPv6 address. + * Legacy IPv6 address may be a normal IPv6 address, or another address + * (e.g. IPv4) disguised as IPv6. This encoding is used in the legacy + * `addr` encoding. + */ + void SetLegacyIPv6(const uint8_t ipv6[16]); + + /** + * Set raw IPv4 or IPv6 address (in network byte order) + * @note Only NET_IPV4 and NET_IPV6 are allowed for network. + */ void SetRaw(Network network, const uint8_t* data); /** @@ -127,7 +142,27 @@ class CNetAddr friend bool operator!=(const CNetAddr& a, const CNetAddr& b); friend bool operator<(const CNetAddr& a, const CNetAddr& b); - SERIALIZE_METHODS(CNetAddr, obj) { READWRITE(obj.ip); } + /** + * Serialize to a stream. + */ + template + void Serialize(Stream& s) const + { + s << ip; + } + + /** + * Unserialize from a stream. + */ + template + void Unserialize(Stream& s) + { + unsigned char ip_temp[sizeof(ip)]; + s >> ip_temp; + // Use SetLegacyIPv6() so that m_net is set correctly. For example + // ::FFFF:0102:0304 should be set as m_net=NET_IPV4 (1.2.3.4). + SetLegacyIPv6(ip_temp); + } friend class CSubNet; }; diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 9f9831c8aabaf..a2d964ab0d78c 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -133,6 +133,14 @@ BOOST_AUTO_TEST_CASE(onioncat_test) BOOST_CHECK(addr1.IsRoutable()); } +BOOST_AUTO_TEST_CASE(embedded_test) +{ + CNetAddr addr1(ResolveIP("1.2.3.4")); + CNetAddr addr2(ResolveIP("::FFFF:0102:0304")); + BOOST_CHECK(addr2.IsIPv4()); + BOOST_CHECK_EQUAL(addr1.ToString(), addr2.ToString()); +} + BOOST_AUTO_TEST_CASE(subnet_test) { @@ -153,9 +161,13 @@ BOOST_AUTO_TEST_CASE(subnet_test) BOOST_CHECK(ResolveSubNet("1.2.2.1/24").Match(ResolveIP("1.2.2.4"))); BOOST_CHECK(ResolveSubNet("1.2.2.110/31").Match(ResolveIP("1.2.2.111"))); BOOST_CHECK(ResolveSubNet("1.2.2.20/26").Match(ResolveIP("1.2.2.63"))); - // All-Matching IPv6 Matches arbitrary IPv4 and IPv6 + // All-Matching IPv6 Matches arbitrary IPv6 BOOST_CHECK(ResolveSubNet("::/0").Match(ResolveIP("1:2:3:4:5:6:7:1234"))); - BOOST_CHECK(ResolveSubNet("::/0").Match(ResolveIP("1.2.3.4"))); + // But not `::` or `0.0.0.0` because they are considered invalid addresses + BOOST_CHECK(!ResolveSubNet("::/0").Match(ResolveIP("::"))); + BOOST_CHECK(!ResolveSubNet("::/0").Match(ResolveIP("0.0.0.0"))); + // Addresses from one network (IPv4) don't belong to subnets of another network (IPv6) + BOOST_CHECK(!ResolveSubNet("::/0").Match(ResolveIP("1.2.3.4"))); // All-Matching IPv4 does not Match IPv6 BOOST_CHECK(!ResolveSubNet("0.0.0.0/0").Match(ResolveIP("1:2:3:4:5:6:7:1234"))); // Invalid subnets Match nothing (not even invalid addresses) From ed5abe1c64009b5ca599996cfa172e6ba4c5c180 Mon Sep 17 00:00:00 2001 From: furszy Date: Sun, 6 Jun 2021 12:36:36 -0300 Subject: [PATCH 09/58] Net: Proper CService deserialization + GetIn6Addr return false if addr isn't an IPv6 addr --- src/netaddress.cpp | 15 ++++++++++++++- src/netaddress.h | 6 +++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 07e1be2473636..86e2083b5ab0b 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -318,7 +318,7 @@ bool operator==(const CNetAddr& a, const CNetAddr& b) bool operator!=(const CNetAddr& a, const CNetAddr& b) { - return (memcmp(a.ip, b.ip, 16) != 0); + return a.m_net != b.m_net || (memcmp(a.ip, b.ip, 16) != 0); } bool operator<(const CNetAddr& a, const CNetAddr& b) @@ -334,8 +334,21 @@ bool CNetAddr::GetInAddr(struct in_addr* pipv4Addr) const return true; } +/** + * Try to get our IPv6 address. + * + * @param[out] pipv6Addr The in6_addr struct to which to copy. + * + * @returns Whether or not the operation was successful, in particular, whether + * or not our address was an IPv6 address. + * + * @see CNetAddr::IsIPv6() + */ bool CNetAddr::GetIn6Addr(struct in6_addr* pipv6Addr) const { + if (!IsIPv6()) { + return false; + } memcpy(pipv6Addr, ip, 16); return true; } diff --git a/src/netaddress.h b/src/netaddress.h index a24c21577b017..93ee4f744e305 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -223,7 +223,11 @@ class CService : public CNetAddr CService(const struct in6_addr& ipv6Addr, uint16_t port); CService(const struct sockaddr_in6& addr); - SERIALIZE_METHODS(CService, obj) { READWRITE(obj.ip, Using>(obj.port)); } + SERIALIZE_METHODS(CService, obj) + { + READWRITEAS(CNetAddr, obj); + READWRITE(Using>(obj.port)); + } }; #endif // PIVX_NETADDRESS_H From f30869d00fe3fc720d563d91ecd143fda62ba7aa Mon Sep 17 00:00:00 2001 From: Mark Tyneway Date: Mon, 16 Dec 2019 19:43:15 -0800 Subject: [PATCH 10/58] test: add IsRFC2544 tests --- src/test/netbase_tests.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index a2d964ab0d78c..e49d67262c407 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -54,6 +54,8 @@ BOOST_AUTO_TEST_CASE(netbase_properties) BOOST_CHECK(ResolveIP("10.0.0.1").IsRFC1918()); BOOST_CHECK(ResolveIP("192.168.1.1").IsRFC1918()); BOOST_CHECK(ResolveIP("172.31.255.255").IsRFC1918()); + BOOST_CHECK(ResolveIP("198.18.0.0").IsRFC2544()); + BOOST_CHECK(ResolveIP("198.19.255.255").IsRFC2544()); BOOST_CHECK(ResolveIP("2001:0DB8::").IsRFC3849()); BOOST_CHECK(ResolveIP("169.254.1.1").IsRFC3927()); BOOST_CHECK(ResolveIP("2002::1").IsRFC3964()); From a751b9bf45be764a9e19f40530bd1cea4d08d017 Mon Sep 17 00:00:00 2001 From: furszy Date: Sun, 6 Jun 2021 13:34:21 -0300 Subject: [PATCH 11/58] net: Avoid using C-style NUL-terminated strings as arguments in the netbase interface Adaptation of btc@9574de86ad703ad942cdd0eca79f48c0d42b102b --- src/init.cpp | 12 +++---- src/net.cpp | 8 ++--- src/netbase.cpp | 68 ++++++++++++++++++++++++-------------- src/netbase.h | 12 +++---- src/qt/clientmodel.cpp | 2 +- src/rpc/net.cpp | 4 +-- src/rpc/rpcevo.cpp | 2 +- src/test/netbase_tests.cpp | 2 +- 8 files changed, 64 insertions(+), 46 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index ab52f44a272d6..ef3fbaa24faa6 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1390,7 +1390,7 @@ bool AppInitMain() for (const auto& net : gArgs.GetArgs("-whitelist")) { CSubNet subnet; - LookupSubNet(net.c_str(), subnet); + LookupSubNet(net, subnet); if (!subnet.IsValid()) return UIError(strprintf(_("Invalid netmask specified in %s: '%s'"), "-whitelist", net)); connman.AddWhitelistedRange(subnet); @@ -1406,7 +1406,7 @@ bool AppInitMain() SetLimited(NET_ONION); if (!proxyArg.empty() && proxyArg != "0") { CService proxyAddr; - if (!Lookup(proxyArg.c_str(), proxyAddr, 9050, fNameLookup)) { + if (!Lookup(proxyArg, proxyAddr, 9050, fNameLookup)) { return UIError(strprintf(_("%s Invalid %s address or hostname: '%s'"), "Lookup():", "-proxy", proxyArg)); } @@ -1430,7 +1430,7 @@ bool AppInitMain() SetLimited(NET_ONION); // set onions as unreachable } else { CService onionProxy; - if (!Lookup(onionArg.c_str(), onionProxy, 9050, fNameLookup)) { + if (!Lookup(onionArg, onionProxy, 9050, fNameLookup)) { return UIError(strprintf(_("%s Invalid %s address or hostname: '%s'"), "Lookup():", "-onion", onionArg)); } proxyType addrOnion = proxyType(onionProxy, proxyRandomize); @@ -1449,13 +1449,13 @@ bool AppInitMain() if (fListen) { for (const std::string& strBind : gArgs.GetArgs("-bind")) { CService addrBind; - if (!Lookup(strBind.c_str(), addrBind, GetListenPort(), false)) + if (!Lookup(strBind, addrBind, GetListenPort(), false)) return UIError(ResolveErrMsg("bind", strBind)); fBound |= Bind(connman, addrBind, (BF_EXPLICIT | BF_REPORT_ERROR)); } for (const std::string& strBind : gArgs.GetArgs("-whitebind")) { CService addrBind; - if (!Lookup(strBind.c_str(), addrBind, 0, false)) + if (!Lookup(strBind, addrBind, 0, false)) return UIError(ResolveErrMsg("whitebind", strBind)); if (addrBind.GetPort() == 0) return UIError(strprintf(_("Need to specify a port with %s: '%s'"), "-whitebind", strBind)); @@ -1473,7 +1473,7 @@ bool AppInitMain() for (const std::string& strAddr : gArgs.GetArgs("-externalip")) { CService addrLocal; - if (Lookup(strAddr.c_str(), addrLocal, GetListenPort(), fNameLookup) && addrLocal.IsValid()) + if (Lookup(strAddr, addrLocal, GetListenPort(), fNameLookup) && addrLocal.IsValid()) AddLocal(addrLocal, LOCAL_MANUAL); else return UIError(ResolveErrMsg("externalip", strAddr)); diff --git a/src/net.cpp b/src/net.cpp index 285adc795990b..e81e134d8c8db 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1400,7 +1400,7 @@ void CConnman::ThreadDNSAddressSeed() if (!resolveSource.SetInternal(host)) { continue; } - if (LookupHost(host.c_str(), vIPs, 0, true)) { + if (LookupHost(host, vIPs, 0, true)) { for (CNetAddr& ip : vIPs) { int nOneDay = 24*3600; CAddress addr = CAddress(CService(ip, Params().GetDefaultPort()), requiredServiceBits); @@ -1641,7 +1641,7 @@ std::vector CConnman::GetAddedNodeInfo() } for (std::string& strAddNode : lAddresses) { - CService service(LookupNumeric(strAddNode.c_str(), Params().GetDefaultPort())); + CService service(LookupNumeric(strAddNode, Params().GetDefaultPort())); if (service.IsValid()) { // strAddNode is an IP:port auto it = mapConnected.find(service); @@ -1678,7 +1678,7 @@ void CConnman::ThreadOpenAddedConnections() CSemaphoreGrant grant(*semOutbound); // If strAddedNode is an IP/port, decode it immediately, so // OpenNetworkConnection can detect existing connections to that IP/port. - CService service(LookupNumeric(info.strAddedNode.c_str(), Params().GetDefaultPort())); + CService service(LookupNumeric(info.strAddedNode, Params().GetDefaultPort())); OpenNetworkConnection(CAddress(service, NODE_NONE), false, &grant, info.strAddedNode.c_str(), false); if (!interruptNet.sleep_for(std::chrono::milliseconds(500))) return; @@ -2461,7 +2461,7 @@ CNode* CConnman::ConnectNode(CAddress addrConnect) bool validateMasternodeIP(const std::string& addrStr) { CNetAddr resolved; - if (LookupHost(addrStr.c_str(), resolved, false)) { + if (LookupHost(addrStr, resolved, false)) { return ((IsReachable(resolved) && resolved.IsRoutable()) || (Params().IsRegTestNet() && resolved.IsValid())); } diff --git a/src/netbase.cpp b/src/netbase.cpp index 0e2ec97cc5dfc..274272c367d45 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -1,15 +1,14 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2015 The Bitcoin developers // Copyright (c) 2017-2020 The PIVX developers -// Distributed under the MIT/X11 software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://www.opensource.org/licenses/mit-license.php. #include "netbase.h" -#include "hash.h" #include "sync.h" -#include "uint256.h" #include "random.h" +#include "util/string.h" #include "util/system.h" #include "utilstrencodings.h" @@ -81,13 +80,17 @@ void SplitHostPort(std::string in, int& portOut, std::string& hostOut) hostOut = in; } -bool static LookupIntern(const char* pszName, std::vector& vIP, unsigned int nMaxSolutions, bool fAllowLookup) +bool static LookupIntern(const std::string& name, std::vector& vIP, unsigned int nMaxSolutions, bool fAllowLookup) { vIP.clear(); + if (!ValidAsCString(name)) { + return false; + } + { CNetAddr addr; - if (addr.SetSpecial(std::string(pszName))) { + if (addr.SetSpecial(name)) { vIP.push_back(addr); return true; } @@ -105,7 +108,7 @@ bool static LookupIntern(const char* pszName, std::vector& vIP, unsign aiHint.ai_flags = fAllowLookup ? AI_ADDRCONFIG : AI_NUMERICHOST; #endif struct addrinfo* aiRes = NULL; - int nErr = getaddrinfo(pszName, NULL, &aiHint, &aiRes); + int nErr = getaddrinfo(name.c_str(), NULL, &aiHint, &aiRes); if (nErr) return false; @@ -135,38 +138,45 @@ bool static LookupIntern(const char* pszName, std::vector& vIP, unsign return (vIP.size() > 0); } -bool LookupHost(const char* pszName, std::vector& vIP, unsigned int nMaxSolutions, bool fAllowLookup) +bool LookupHost(const std::string& name, std::vector& vIP, unsigned int nMaxSolutions, bool fAllowLookup) { - std::string strHost(pszName); + if (!ValidAsCString(name)) { + return false; + } + std::string strHost = name; if (strHost.empty()) return false; if (strHost.front() == '[' && strHost.back() == ']') { strHost = strHost.substr(1, strHost.size() - 2); } - return LookupIntern(strHost.c_str(), vIP, nMaxSolutions, fAllowLookup); + return LookupIntern(strHost, vIP, nMaxSolutions, fAllowLookup); } -bool LookupHost(const char* pszName, CNetAddr& addr, bool fAllowLookup) +bool LookupHost(const std::string& name, CNetAddr& addr, bool fAllowLookup) { + if (!ValidAsCString(name)) { + return false; + } std::vector vIP; - LookupHost(pszName, vIP, 1, fAllowLookup); + LookupHost(name, vIP, 1, fAllowLookup); if (vIP.empty()) return false; addr = vIP.front(); return true; } -bool Lookup(const char* pszName, std::vector& vAddr, int portDefault, bool fAllowLookup, unsigned int nMaxSolutions) +bool Lookup(const std::string& name, std::vector& vAddr, int portDefault, bool fAllowLookup, unsigned int nMaxSolutions) { - if (pszName[0] == 0) + if (name.empty() || !ValidAsCString(name)) { return false; + } int port = portDefault; - std::string hostname = ""; - SplitHostPort(std::string(pszName), port, hostname); + std::string hostname; + SplitHostPort(name, port, hostname); std::vector vIP; - bool fRet = LookupIntern(hostname.c_str(), vIP, nMaxSolutions, fAllowLookup); + bool fRet = LookupIntern(hostname, vIP, nMaxSolutions, fAllowLookup); if (!fRet) return false; vAddr.resize(vIP.size()); @@ -175,22 +185,28 @@ bool Lookup(const char* pszName, std::vector& vAddr, int portDefault, return true; } -bool Lookup(const char* pszName, CService& addr, int portDefault, bool fAllowLookup) +bool Lookup(const std::string& name, CService& addr, int portDefault, bool fAllowLookup) { + if (!ValidAsCString(name)) { + return false; + } std::vector vService; - bool fRet = Lookup(pszName, vService, portDefault, fAllowLookup, 1); + bool fRet = Lookup(name, vService, portDefault, fAllowLookup, 1); if (!fRet) return false; addr = vService[0]; return true; } -CService LookupNumeric(const char* pszName, int portDefault) +CService LookupNumeric(const std::string& name, int portDefault) { + if (!ValidAsCString(name)) { + return {}; + } CService addr; // "1.2:345" will fail to resolve the ip, but will still set the port. // If the ip fails to resolve, re-init the result. - if (!Lookup(pszName, addr, portDefault, false)) + if (!Lookup(name, addr, portDefault, false)) addr = CService(); return addr; } @@ -669,14 +685,16 @@ bool ConnectSocketByName(CService& addr, SOCKET& hSocketRet, const char* pszDest return ConnectThroughProxy(proxy, strDest, port, hSocketRet, nTimeout, outProxyConnectionFailed); } -bool LookupSubNet(const char* pszName, CSubNet& ret) +bool LookupSubNet(const std::string& strSubnet, CSubNet& ret) { - std::string strSubnet(pszName); + if (!ValidAsCString(strSubnet)) { + return false; + } size_t slash = strSubnet.find_last_of('/'); std::vector vIP; std::string strAddress = strSubnet.substr(0, slash); - if (LookupHost(strAddress.c_str(), vIP, 1, false)) + if (LookupHost(strAddress, vIP, 1, false)) { CNetAddr network = vIP[0]; if (slash != strSubnet.npos) { @@ -689,7 +707,7 @@ bool LookupSubNet(const char* pszName, CSubNet& ret) } else // If not a valid number, try full netmask syntax { // Never allow lookup for netmask - if (LookupHost(strNetmask.c_str(), vIP, 1, false)) { + if (LookupHost(strNetmask, vIP, 1, false)) { ret = CSubNet(network, vIP[0]); return ret.IsValid(); } diff --git a/src/netbase.h b/src/netbase.h index 06bad8f300e06..8d124c1e82422 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -46,12 +46,12 @@ bool GetProxy(enum Network net, proxyType& proxyInfoOut); bool IsProxy(const CNetAddr& addr); bool SetNameProxy(const proxyType &addrProxy); bool HaveNameProxy(); -bool LookupHost(const char* pszName, std::vector& vIP, unsigned int nMaxSolutions, bool fAllowLookup); -bool LookupHost(const char* pszName, CNetAddr& addr, bool fAllowLookup); -bool Lookup(const char* pszName, CService& addr, int portDefault, bool fAllowLookup); -bool Lookup(const char* pszName, std::vector& vAddr, int portDefault, bool fAllowLookup, unsigned int nMaxSolutions); -CService LookupNumeric(const char* pszName, int portDefault = 0); -bool LookupSubNet(const char* pszName, CSubNet& subnet); +bool LookupHost(const std::string& name, std::vector& vIP, unsigned int nMaxSolutions, bool fAllowLookup); +bool LookupHost(const std::string& name, CNetAddr& addr, bool fAllowLookup); +bool Lookup(const std::string& name, CService& addr, int portDefault, bool fAllowLookup); +bool Lookup(const std::string& name, std::vector& vAddr, int portDefault, bool fAllowLookup, unsigned int nMaxSolutions); +CService LookupNumeric(const std::string& name, int portDefault = 0); +bool LookupSubNet(const std::string& name, CSubNet& subnet); bool ConnectSocket(const CService& addr, SOCKET& hSocketRet, int nTimeout, bool* outProxyConnectionFailed = 0); bool ConnectSocketByName(CService& addr, SOCKET& hSocketRet, const char* pszDest, int portDefault, int nTimeout, bool* outProxyConnectionFailed = 0); /** Return readable error string for a network error code */ diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index fff6ae476f39f..785983ffaa5b3 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -364,7 +364,7 @@ bool ClientModel::getTorInfo(std::string& ip_port) const LOCK(cs_mapLocalHost); for (const std::pair& item : mapLocalHost) { if (item.first.IsTor()) { - CService addrOnion(LookupNumeric(item.first.ToString().c_str(), item.second.nPort)); + CService addrOnion(LookupNumeric(item.first.ToString(), item.second.nPort)); ip_port = addrOnion.ToStringIPPort(); return true; } diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 8b4ebd7926ebe..a375d1da5a502 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -466,10 +466,10 @@ UniValue setban(const JSONRPCRequest& request) if (!isSubnet) { CNetAddr resolved; - LookupHost(request.params[0].get_str().c_str(), resolved, false); + LookupHost(request.params[0].get_str(), resolved, false); netAddr = resolved; } else - LookupSubNet(request.params[0].get_str().c_str(), subNet); + LookupSubNet(request.params[0].get_str(), subNet); if (! (isSubnet ? subNet.IsValid() : netAddr.IsValid()) ) throw JSONRPCError(RPC_CLIENT_NODE_ALREADY_ADDED, "Error: Invalid IP/Subnet"); diff --git a/src/rpc/rpcevo.cpp b/src/rpc/rpcevo.cpp index 9f8770d0efcac..a68326038cf99 100644 --- a/src/rpc/rpcevo.cpp +++ b/src/rpc/rpcevo.cpp @@ -330,7 +330,7 @@ static ProRegPL ParseProRegPLParams(const UniValue& params, unsigned int paramId // ip and port const std::string& strIpPort = params[paramIdx].get_str(); if (!strIpPort.empty()) { - if (!Lookup(strIpPort.c_str(), pl.addr, Params().GetDefaultPort(), false)) { + if (!Lookup(strIpPort, pl.addr, Params().GetDefaultPort(), false)) { throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("invalid network address %s", strIpPort)); } } diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index e49d67262c407..9af26b7cd90a9 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -104,7 +104,7 @@ BOOST_AUTO_TEST_CASE(netbase_splithost) bool static TestParse(std::string src, std::string canon) { - CService addr(LookupNumeric(src.c_str(), 65535)); + CService addr(LookupNumeric(src, 65535)); return canon == addr.ToString(); } From f6c52a322ef2ca16d114796d35ec8e25057f7231 Mon Sep 17 00:00:00 2001 From: practicalswift Date: Mon, 16 Dec 2019 11:46:19 +0000 Subject: [PATCH 12/58] tests: Add tests to make sure lookup methods fail on std::string parameters with embedded NUL characters --- src/test/net_tests.cpp | 8 +++++--- src/test/netbase_tests.cpp | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index d51dea6b9d540..f00adbed952ad 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -96,9 +96,11 @@ BOOST_AUTO_TEST_CASE(caddrdb_read) addrmanUncorrupted.MakeDeterministic(); CService addr1, addr2, addr3; - Lookup("250.7.1.1", addr1, 8333, false); - Lookup("250.7.2.2", addr2, 9999, false); - Lookup("250.7.3.3", addr3, 9999, false); + BOOST_CHECK(Lookup("250.7.1.1", addr1, 8333, false)); + BOOST_CHECK(Lookup("250.7.2.2", addr2, 9999, false)); + BOOST_CHECK(Lookup("250.7.3.3", addr3, 9999, false)); + BOOST_CHECK(Lookup(std::string("250.7.3.3", 9), addr3, 9999, false)); + BOOST_CHECK(!Lookup(std::string("250.7.3.3\0example.com", 21), addr3, 9999, false)); // Add three addresses to new table. CService source; diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 9af26b7cd90a9..f1e10a096550f 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -357,4 +357,22 @@ BOOST_AUTO_TEST_CASE(netbase_parsenetwork) BOOST_CHECK_EQUAL(ParseNetwork(""), NET_UNROUTABLE); } +BOOST_AUTO_TEST_CASE(netbase_dont_resolve_strings_with_embedded_nul_characters) +{ + CNetAddr addr; + BOOST_CHECK(LookupHost(std::string("127.0.0.1", 9), addr, false)); + BOOST_CHECK(!LookupHost(std::string("127.0.0.1\0", 10), addr, false)); + BOOST_CHECK(!LookupHost(std::string("127.0.0.1\0example.com", 21), addr, false)); + BOOST_CHECK(!LookupHost(std::string("127.0.0.1\0example.com\0", 22), addr, false)); + CSubNet ret; + BOOST_CHECK(LookupSubNet(std::string("1.2.3.0/24", 10), ret)); + BOOST_CHECK(!LookupSubNet(std::string("1.2.3.0/24\0", 11), ret)); + BOOST_CHECK(!LookupSubNet(std::string("1.2.3.0/24\0example.com", 22), ret)); + BOOST_CHECK(!LookupSubNet(std::string("1.2.3.0/24\0example.com\0", 23), ret)); + BOOST_CHECK(LookupSubNet(std::string("5wyqrzbvrdsumnok.onion", 22), ret)); + BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0", 23), ret)); + BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0example.com", 34), ret)); + BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0example.com\0", 35), ret)); +} + BOOST_AUTO_TEST_SUITE_END() From 9f9c871a8e1b4ac5058dd14879b91c598c642110 Mon Sep 17 00:00:00 2001 From: practicalswift Date: Sun, 29 Dec 2019 20:23:48 +0000 Subject: [PATCH 13/58] tests: Avoid using C-style NUL-terminated strings as arguments --- src/test/addrman_tests.cpp | 13 ++----------- src/test/netbase_tests.cpp | 6 +++--- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index 8f71c88868642..ba3eb77b513e1 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -93,29 +93,20 @@ class CAddrManTest : public CAddrMan }; -static CNetAddr ResolveIP(const char* ip) +static CNetAddr ResolveIP(const std::string& ip) { CNetAddr addr; BOOST_CHECK_MESSAGE(LookupHost(ip, addr, false), strprintf("failed to resolve: %s", ip)); return addr; } -static CNetAddr ResolveIP(std::string ip) -{ - return ResolveIP(ip.c_str()); -} - -static CService ResolveService(const char* ip, int port = 0) +static CService ResolveService(const std::string& ip, const int port = 0) { CService serv; BOOST_CHECK_MESSAGE(Lookup(ip, serv, port, false), strprintf("failed to resolve: %s:%i", ip, port)); return serv; } -static CService ResolveService(std::string ip, int port = 0) -{ - return ResolveService(ip.c_str(), port); -} static std::vector FromBytes(const unsigned char* source, int vector_size) { std::vector result(vector_size); diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index f1e10a096550f..eb03d24316b0f 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -15,21 +15,21 @@ BOOST_FIXTURE_TEST_SUITE(netbase_tests, BasicTestingSetup) -static CNetAddr ResolveIP(const char* ip) +static CNetAddr ResolveIP(const std::string& ip) { CNetAddr addr; LookupHost(ip, addr, false); return addr; } -static CSubNet ResolveSubNet(const char* subnet) +static CSubNet ResolveSubNet(const std::string& subnet) { CSubNet ret; LookupSubNet(subnet, ret); return ret; } -static CNetAddr CreateInternal(const char* host) +static CNetAddr CreateInternal(const std::string& host) { CNetAddr addr; addr.SetInternal(host); From 31064a8f6679574d1b90f45316fe529ec1594ebd Mon Sep 17 00:00:00 2001 From: furszy Date: Sun, 6 Jun 2021 14:09:20 -0300 Subject: [PATCH 14/58] net: Minor accumulated cleanups Adaptation of btc@2c084a6609bed24979c2a144743007f8b10a5c70 --- src/compat.h | 6 ++++++ src/net.cpp | 22 ++++++++++------------ src/netaddress.cpp | 15 ++------------- src/netaddress.h | 4 +--- src/netbase.cpp | 6 +----- src/pivxd.cpp | 5 +---- 6 files changed, 21 insertions(+), 37 deletions(-) diff --git a/src/compat.h b/src/compat.h index f51f9df965201..f0c40f67d9d76 100644 --- a/src/compat.h +++ b/src/compat.h @@ -93,6 +93,12 @@ typedef u_int SOCKET; size_t strnlen( const char *start, size_t max_len); #endif // HAVE_DECL_STRNLEN +#ifndef WIN32 +typedef void* sockopt_arg_type; +#else +typedef char* sockopt_arg_type; +#endif + bool static inline IsSelectableSocket(SOCKET s) { #ifdef WIN32 diff --git a/src/net.cpp b/src/net.cpp index e81e134d8c8db..e07f5c12e776c 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1640,25 +1640,27 @@ std::vector CConnman::GetAddedNodeInfo() } } - for (std::string& strAddNode : lAddresses) { + for (const std::string& strAddNode : lAddresses) { CService service(LookupNumeric(strAddNode, Params().GetDefaultPort())); + AddedNodeInfo addedNode{strAddNode, CService(), false, false}; if (service.IsValid()) { // strAddNode is an IP:port auto it = mapConnected.find(service); if (it != mapConnected.end()) { - ret.emplace_back(strAddNode, service, true, it->second); - } else { - ret.emplace_back(strAddNode, CService(), false, false); + addedNode.resolvedAddress = service; + addedNode.fConnected = true; + addedNode.fInbound = it->second; } } else { // strAddNode is a name auto it = mapConnectedByName.find(strAddNode); if (it != mapConnectedByName.end()) { - ret.emplace_back(strAddNode, it->second.second, true, it->second.first); - } else { - ret.emplace_back(strAddNode, CService(), false, false); + addedNode.resolvedAddress = it->second.second; + addedNode.fConnected = true; + addedNode.fInbound = it->second.first; } } + ret.emplace_back(std::move(addedNode)); } return ret; @@ -1824,11 +1826,7 @@ bool CConnman::BindListenPort(const CService& addrBind, std::string& strError, b // and enable it by default or not. Try to enable it, if possible. if (addrBind.IsIPv6()) { #ifdef IPV6_V6ONLY -#ifdef WIN32 - setsockopt(hListenSocket, IPPROTO_IPV6, IPV6_V6ONLY, (const char*)&nOne, sizeof(int)); -#else - setsockopt(hListenSocket, IPPROTO_IPV6, IPV6_V6ONLY, (void*)&nOne, sizeof(int)); -#endif + setsockopt(hListenSocket, IPPROTO_IPV6, IPV6_V6ONLY, (sockopt_arg_type)&nOne, sizeof(int)); #endif #ifdef WIN32 int nProtLevel = PROTECTION_LEVEL_UNRESTRICTED; diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 86e2083b5ab0b..28b58013a9cc2 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -18,7 +18,7 @@ static const unsigned char pchOnionCat[] = {0xFD,0x87,0xD8,0x7E,0xEB,0x43}; // 0xFD + sha256("bitcoin")[0:5] static const unsigned char g_internal_prefix[] = { 0xFD, 0x6B, 0x88, 0xC0, 0x87, 0x24 }; -void CNetAddr::Init() +CNetAddr::CNetAddr() { memset(ip, 0, sizeof(ip)); } @@ -88,11 +88,6 @@ bool CNetAddr::SetSpecial(const std::string& strName) return false; } -CNetAddr::CNetAddr() -{ - Init(); -} - CNetAddr::CNetAddr(const struct in_addr& ipv4Addr) { SetRaw(NET_IPV4, (const uint8_t*)&ipv4Addr); @@ -569,14 +564,8 @@ int CNetAddr::GetReachabilityFrom(const CNetAddr *paddrPartner) const } } -void CService::Init() -{ - port = 0; -} - -CService::CService() +CService::CService() : port(0) { - Init(); } CService::CService(const CNetAddr& cip, uint16_t portIn) : CNetAddr(cip), port(portIn) diff --git a/src/netaddress.h b/src/netaddress.h index 93ee4f744e305..a1196f1f9e9b8 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -67,8 +67,7 @@ class CNetAddr public: CNetAddr(); - CNetAddr(const struct in_addr& ipv4Addr); - void Init(); + explicit CNetAddr(const struct in_addr& ipv4Addr); void SetIP(const CNetAddr& ip); /** @@ -208,7 +207,6 @@ class CService : public CNetAddr CService(const CNetAddr& ip, uint16_t port); CService(const struct in_addr& ipv4Addr, uint16_t port); CService(const struct sockaddr_in& addr); - void Init(); uint16_t GetPort() const; bool GetSockAddr(struct sockaddr* paddr, socklen_t* addrlen) const; bool SetSockAddr(const struct sockaddr* paddr); diff --git a/src/netbase.cpp b/src/netbase.cpp index 274272c367d45..75e8cb72d7a8c 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -532,11 +532,7 @@ bool static ConnectSocketDirectly(const CService& addrConnect, SOCKET& hSocketRe return false; } socklen_t nRetSize = sizeof(nRet); -#ifdef WIN32 - if (getsockopt(hSocket, SOL_SOCKET, SO_ERROR, (char*)(&nRet), &nRetSize) == SOCKET_ERROR) -#else - if (getsockopt(hSocket, SOL_SOCKET, SO_ERROR, &nRet, &nRetSize) == SOCKET_ERROR) -#endif + if (getsockopt(hSocket, SOL_SOCKET, SO_ERROR, (sockopt_arg_type)&nRet, &nRetSize) == SOCKET_ERROR) { LogPrintf("getsockopt() for %s failed: %s\n", addrConnect.ToString(), NetworkErrorString(WSAGetLastError())); CloseSocket(hSocket); diff --git a/src/pivxd.cpp b/src/pivxd.cpp index 208f3b39066db..fc96a5e1b1a89 100644 --- a/src/pivxd.cpp +++ b/src/pivxd.cpp @@ -35,11 +35,8 @@ static bool fDaemon; void WaitForShutdown() { - bool fShutdown = ShutdownRequested(); - // Tell the main threads to shutdown. - while (!fShutdown) { + while (!ShutdownRequested()) { MilliSleep(200); - fShutdown = ShutdownRequested(); } Interrupt(); } From 4fdfa45ae2cf43764143d0d1d66b6555abd26206 Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Sun, 6 Jun 2021 23:52:40 -0300 Subject: [PATCH 15/58] net: Always default rpcbind to localhost, never "all interfaces" We don't support binding to untrusted networks, so avoid a default where that is typical --- src/httpserver.cpp | 8 ++++---- src/init.cpp | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 2dbf0b1ce5511..3245373ae0027 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -304,9 +304,12 @@ static bool HTTPBindAddresses(struct evhttp* http) std::vector > endpoints; // Determine what addresses to bind to - if (!gArgs.IsArgSet("-rpcallowip")) { // Default to loopback if not allowing external IPs + if (!(gArgs.IsArgSet("-rpcallowip") && gArgs.IsArgSet("-rpcbind"))) { // Default to loopback if not allowing external IPs endpoints.emplace_back("::1", defaultPort); endpoints.emplace_back("127.0.0.1", defaultPort); + if (gArgs.IsArgSet("-rpcallowip")) { + LogPrintf("WARNING: option -rpcallowip was specified without -rpcbind; this doesn't usually make sense\n"); + } if (gArgs.IsArgSet("-rpcbind")) { LogPrintf("WARNING: option -rpcbind was ignored because -rpcallowip was not specified, refusing to allow everyone to connect\n"); } @@ -317,9 +320,6 @@ static bool HTTPBindAddresses(struct evhttp* http) SplitHostPort(strRPCBind, port, host); endpoints.emplace_back(host, port); } - } else { // No specific bind address specified, bind to any - endpoints.emplace_back("::", defaultPort); - endpoints.emplace_back("0.0.0.0", defaultPort); } // Bind addresses diff --git a/src/init.cpp b/src/init.cpp index ef3fbaa24faa6..d51eec6075ceb 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -600,7 +600,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageGroup(_("RPC server options:")); strUsage += HelpMessageOpt("-server", _("Accept command line and JSON-RPC commands")); strUsage += HelpMessageOpt("-rest", strprintf(_("Accept public REST requests (default: %u)"), DEFAULT_REST_ENABLE)); - strUsage += HelpMessageOpt("-rpcbind=", _("Bind to given address to listen for JSON-RPC connections. Use [host]:port notation for IPv6. This option can be specified multiple times (default: bind to all interfaces)")); + strUsage += HelpMessageOpt("-rpcbind=", _("Bind to given address to listen for JSON-RPC connections. This option is ignored unless -rpcallowip is also passed. Port is optional and overrides -rpcport. Use [host]:port notation for IPv6. This option can be specified multiple times (default: 127.0.0.1 and ::1 i.e., localhost)")); strUsage += HelpMessageOpt("-rpccookiefile=", _("Location of the auth cookie (default: data dir)")); strUsage += HelpMessageOpt("-rpcuser=", _("Username for JSON-RPC connections")); strUsage += HelpMessageOpt("-rpcpassword=", _("Password for JSON-RPC connections")); From 2a6abd8080237e07db9ec92d734166cfc0b13938 Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Sat, 20 Oct 2018 14:48:29 +0000 Subject: [PATCH 16/58] CNetAddr: Add IsBindAny method to check for INADDR_ANY --- src/netaddress.cpp | 10 ++++++++++ src/netaddress.h | 1 + 2 files changed, 11 insertions(+) diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 28b58013a9cc2..226af30dbd6fe 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -104,6 +104,16 @@ unsigned int CNetAddr::GetByte(int n) const return ip[15-n]; } +bool CNetAddr::IsBindAny() const +{ + const int cmplen = IsIPv4() ? 4 : 16; + for (int i = 0; i < cmplen; ++i) { + if (GetByte(i)) return false; + } + + return true; +} + bool CNetAddr::IsIPv4() const { return m_net == NET_IPV4; } bool CNetAddr::IsIPv6() const { return m_net == NET_IPV6; } diff --git a/src/netaddress.h b/src/netaddress.h index a1196f1f9e9b8..770a7b1ef760e 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -91,6 +91,7 @@ class CNetAddr bool SetInternal(const std::string& name); bool SetSpecial(const std::string& strName); // for Tor addresses + bool IsBindAny() const; // INADDR_ANY equivalent bool IsIPv4() const; // IPv4 mapped address (::FFFF:0:0/96, 0.0.0.0/0) bool IsIPv6() const; // IPv6 address (not mapped IPv4, not Tor) bool IsRFC1918() const; // IPv4 private networks (10.0.0.0/8, 192.168.0.0/16, 172.16.0.0/12) From 5d7f86441935b76d5e7e23ead54c464080741d0b Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Sun, 6 Jun 2021 23:55:18 -0300 Subject: [PATCH 17/58] rpcbind: Warn about exposing RPC to untrusted networks --- src/httpserver.cpp | 4 ++++ src/init.cpp | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 3245373ae0027..de716e97691ba 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -327,6 +327,10 @@ static bool HTTPBindAddresses(struct evhttp* http) LogPrint(BCLog::HTTP, "Binding RPC on address %s port %i\n", i->first, i->second); evhttp_bound_socket *bind_handle = evhttp_bind_socket_with_handle(http, i->first.empty() ? NULL : i->first.c_str(), i->second); if (bind_handle) { + CNetAddr addr; + if (i->first.empty() || (LookupHost(i->first, addr, false) && addr.IsBindAny())) { + LogPrintf("WARNING: the RPC server is not safe to expose to untrusted networks such as the public internet\n"); + } boundSockets.push_back(bind_handle); } else { LogPrintf("Binding RPC on address %s port %i failed.\n", i->first, i->second); diff --git a/src/init.cpp b/src/init.cpp index d51eec6075ceb..b7a0b5f2fd7c1 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -600,7 +600,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageGroup(_("RPC server options:")); strUsage += HelpMessageOpt("-server", _("Accept command line and JSON-RPC commands")); strUsage += HelpMessageOpt("-rest", strprintf(_("Accept public REST requests (default: %u)"), DEFAULT_REST_ENABLE)); - strUsage += HelpMessageOpt("-rpcbind=", _("Bind to given address to listen for JSON-RPC connections. This option is ignored unless -rpcallowip is also passed. Port is optional and overrides -rpcport. Use [host]:port notation for IPv6. This option can be specified multiple times (default: 127.0.0.1 and ::1 i.e., localhost)")); + strUsage += HelpMessageOpt("-rpcbind=", _("Bind to given address to listen for JSON-RPC connections. Do not expose the RPC server to untrusted networks such as the public internet! This option is ignored unless -rpcallowip is also passed. Port is optional and overrides -rpcport. Use [host]:port notation for IPv6. This option can be specified multiple times (default: 127.0.0.1 and ::1 i.e., localhost)")); strUsage += HelpMessageOpt("-rpccookiefile=", _("Location of the auth cookie (default: data dir)")); strUsage += HelpMessageOpt("-rpcuser=", _("Username for JSON-RPC connections")); strUsage += HelpMessageOpt("-rpcpassword=", _("Password for JSON-RPC connections")); From 8839828ac9fe8409410e8df4d946dfe5afa0b85e Mon Sep 17 00:00:00 2001 From: Vasil Dimov Date: Mon, 24 Aug 2020 21:03:31 +0200 Subject: [PATCH 18/58] net: don't accept non-left-contiguous netmasks A netmask that contains 1-bits after 0-bits (the 1-bits are not contiguous on the left side) is invalid [1] [2]. The code before this PR used to parse and accept such non-left-contiguous netmasks. However, a coming change that will alter `CNetAddr::ip` to have flexible size would make juggling with such netmasks more difficult, thus drop support for those. [1] https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#Subnet_masks [2] https://tools.ietf.org/html/rfc4632#section-5.1 --- doc/release-notes.md | 7 +++ src/netaddress.cpp | 88 +++++++++++++++++--------------------- src/test/netbase_tests.cpp | 7 ++- 3 files changed, 52 insertions(+), 50 deletions(-) diff --git a/doc/release-notes.md b/doc/release-notes.md index e242b03df8de8..20f9357d0b223 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -533,6 +533,13 @@ The `autocombine` RPC command has been replaced with specific set/get commands ( } ``` +Updated settings +---------------- + +- Netmasks that contain 1-bits after 0-bits (the 1-bits are not contiguous on + the left side, e.g. 255.0.255.255) are no longer accepted. They are invalid + according to RFC 4632. + Build system changes -------------------- diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 226af30dbd6fe..7c0628cafdcb8 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -724,9 +724,41 @@ CSubNet::CSubNet(const CNetAddr &addr, int32_t mask) network.ip[x] &= netmask[x]; } +/** + * @returns The number of 1-bits in the prefix of the specified subnet mask. If + * the specified subnet mask is not a valid one, -1. + */ +static inline int NetmaskBits(uint8_t x) +{ + switch(x) { + case 0x00: return 0; + case 0x80: return 1; + case 0xc0: return 2; + case 0xe0: return 3; + case 0xf0: return 4; + case 0xf8: return 5; + case 0xfc: return 6; + case 0xfe: return 7; + case 0xff: return 8; + default: return -1; + } +} + CSubNet::CSubNet(const CNetAddr &addr, const CNetAddr &mask) { valid = true; + // Check if `mask` contains 1-bits after 0-bits (which is an invalid netmask). + bool zeros_found = false; + for (size_t i = mask.IsIPv4() ? 12 : 0; i < sizeof(mask.ip); ++i) { + const int num_bits = NetmaskBits(mask.ip[i]); + if (num_bits == -1 || (zeros_found && num_bits != 0)) { + valid = false; + return; + } + if (num_bits < 8) { + zeros_found = true; + } + } network = addr; // Default to /32 (IPv4) or /128 (IPv6), i.e. match single address memset(netmask, 255, sizeof(netmask)); @@ -759,58 +791,18 @@ bool CSubNet::Match(const CNetAddr &addr) const return true; } -static inline int NetmaskBits(uint8_t x) -{ - switch(x) { - case 0x00: return 0; break; - case 0x80: return 1; break; - case 0xc0: return 2; break; - case 0xe0: return 3; break; - case 0xf0: return 4; break; - case 0xf8: return 5; break; - case 0xfc: return 6; break; - case 0xfe: return 7; break; - case 0xff: return 8; break; - default: return -1; break; - } -} - std::string CSubNet::ToString() const { - /* Parse binary 1{n}0{N-n} to see if mask can be represented as /n */ - int cidr = 0; - bool valid_cidr = true; - int n = network.IsIPv4() ? 12 : 0; - for (; n < 16 && netmask[n] == 0xff; ++n) - cidr += 8; - if (n < 16) { - int bits = NetmaskBits(netmask[n]); - if (bits < 0) - valid_cidr = false; - else - cidr += bits; - ++n; - } - for (; n < 16 && valid_cidr; ++n) - if (netmask[n] != 0x00) - valid_cidr = false; - - /* Format output */ - std::string strNetmask; - if (valid_cidr) { - strNetmask = strprintf("%u", cidr); - } else { - if (network.IsIPv4()) - strNetmask = strprintf("%u.%u.%u.%u", netmask[12], netmask[13], netmask[14], netmask[15]); - else - strNetmask = strprintf("%x:%x:%x:%x:%x:%x:%x:%x", - netmask[0] << 8 | netmask[1], netmask[2] << 8 | netmask[3], - netmask[4] << 8 | netmask[5], netmask[6] << 8 | netmask[7], - netmask[8] << 8 | netmask[9], netmask[10] << 8 | netmask[11], - netmask[12] << 8 | netmask[13], netmask[14] << 8 | netmask[15]); + uint8_t cidr = 0; + + for (size_t i = network.IsIPv4() ? 12 : 0; i < sizeof(netmask); ++i) { + if (netmask[i] == 0x00) { + break; + } + cidr += NetmaskBits(netmask[i]); } - return network.ToString() + "/" + strNetmask; + return network.ToString() + strprintf("/%u", cidr); } bool CSubNet::IsValid() const diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index eb03d24316b0f..9c12765fc151c 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -287,10 +287,13 @@ BOOST_AUTO_TEST_CASE(subnet_test) BOOST_CHECK_EQUAL(subnet.ToString(), "1::/16"); subnet = ResolveSubNet("1:2:3:4:5:6:7:8/0000:0000:0000:0000:0000:0000:0000:0000"); BOOST_CHECK_EQUAL(subnet.ToString(), "::/0"); + // Invalid netmasks (with 1-bits after 0-bits) subnet = ResolveSubNet("1.2.3.4/255.255.232.0"); - BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.0.0/255.255.232.0"); + BOOST_CHECK(!subnet.IsValid()); + subnet = ResolveSubNet("1.2.3.4/255.0.255.255"); + BOOST_CHECK(!subnet.IsValid()); subnet = ResolveSubNet("1:2:3:4:5:6:7:8/ffff:ffff:ffff:fffe:ffff:ffff:ffff:ff0f"); - BOOST_CHECK_EQUAL(subnet.ToString(), "1:2:3:4:5:6:7:8/ffff:ffff:ffff:fffe:ffff:ffff:ffff:ff0f"); + BOOST_CHECK(!subnet.IsValid()); } BOOST_AUTO_TEST_CASE(validate_test) From a40711b5c67bad088ff6f9d6c5bb3ff95955b814 Mon Sep 17 00:00:00 2001 From: marcaiaf Date: Fri, 28 Dec 2018 16:20:30 +0100 Subject: [PATCH 19/58] IsReachable is the inverse of IsLimited (DRY). Includes unit tests --- src/net.cpp | 6 +-- src/test/net_tests.cpp | 88 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 4 deletions(-) diff --git a/src/net.cpp b/src/net.cpp index e07f5c12e776c..53bfacb4a3dae 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -263,15 +263,13 @@ bool IsLocal(const CService& addr) /** check whether a given network is one we can probably connect to */ bool IsReachable(enum Network net) { - LOCK(cs_mapLocalHost); - return !vfLimited[net]; + return !IsLimited(net); } /** check whether a given address is in a network we can probably connect to */ bool IsReachable(const CNetAddr& addr) { - enum Network net = addr.GetNetwork(); - return IsReachable(net); + return IsReachable(addr.GetNetwork()); } diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index f00adbed952ad..ccb66adfc3571 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -230,6 +230,94 @@ BOOST_AUTO_TEST_CASE(ipv4_peer_with_ipv6_addrMe_test) BOOST_CHECK(1); } + +BOOST_AUTO_TEST_CASE(LimitedAndReachable_Network) +{ + SetLimited(NET_IPV4, true); + SetLimited(NET_IPV6, true); + SetLimited(NET_ONION, true); + + BOOST_CHECK_EQUAL(IsLimited(NET_IPV4), true); + BOOST_CHECK_EQUAL(IsLimited(NET_IPV6), true); + BOOST_CHECK_EQUAL(IsLimited(NET_ONION), true); + + BOOST_CHECK_EQUAL(IsReachable(NET_IPV4), false); + BOOST_CHECK_EQUAL(IsReachable(NET_IPV6), false); + BOOST_CHECK_EQUAL(IsReachable(NET_ONION), false); + + + SetLimited(NET_IPV4, false); + SetLimited(NET_IPV6, false); + SetLimited(NET_ONION, false); + + BOOST_CHECK_EQUAL(IsLimited(NET_IPV4), false); + BOOST_CHECK_EQUAL(IsLimited(NET_IPV6), false); + BOOST_CHECK_EQUAL(IsLimited(NET_ONION), false); + + BOOST_CHECK_EQUAL(IsReachable(NET_IPV4), true); + BOOST_CHECK_EQUAL(IsReachable(NET_IPV6), true); + BOOST_CHECK_EQUAL(IsReachable(NET_ONION), true); +} + +BOOST_AUTO_TEST_CASE(LimitedAndReachable_NetworkCaseUnroutableAndInternal) +{ + BOOST_CHECK_EQUAL(IsLimited(NET_UNROUTABLE), false); + BOOST_CHECK_EQUAL(IsLimited(NET_INTERNAL), false); + + BOOST_CHECK_EQUAL(IsReachable(NET_UNROUTABLE), true); + BOOST_CHECK_EQUAL(IsReachable(NET_INTERNAL), true); + + SetLimited(NET_UNROUTABLE, true); + SetLimited(NET_INTERNAL, true); + + BOOST_CHECK_EQUAL(IsLimited(NET_UNROUTABLE), false); // Ignored for both networks + BOOST_CHECK_EQUAL(IsLimited(NET_INTERNAL), false); + + BOOST_CHECK_EQUAL(IsReachable(NET_UNROUTABLE), true); + BOOST_CHECK_EQUAL(IsReachable(NET_INTERNAL), true); +} + +CNetAddr UtilBuildAddress(unsigned char p1, unsigned char p2, unsigned char p3, unsigned char p4) +{ + unsigned char ip[] = {p1, p2, p3, p4}; + + struct sockaddr_in sa; + memset(&sa, 0, sizeof(sockaddr_in)); // initialize the memory block + memcpy(&(sa.sin_addr), &ip, sizeof(ip)); + return CNetAddr(sa.sin_addr); +} + + +BOOST_AUTO_TEST_CASE(LimitedAndReachable_CNetAddr) +{ + CNetAddr addr = UtilBuildAddress(0x001, 0x001, 0x001, 0x001); // 1.1.1.1 + + SetLimited(NET_IPV4, false); + BOOST_CHECK_EQUAL(IsLimited(addr), false); + BOOST_CHECK_EQUAL(IsReachable(addr), true); + + SetLimited(NET_IPV4, true); + BOOST_CHECK_EQUAL(IsLimited(addr), true); + BOOST_CHECK_EQUAL(IsReachable(addr), false); + + SetLimited(NET_IPV4, false); // have to reset this, because this is stateful. +} + + +BOOST_AUTO_TEST_CASE(LocalAddress_BasicLifecycle) +{ + CService addr = CService(UtilBuildAddress(0x002, 0x001, 0x001, 0x001), 1000); // 2.1.1.1:1000 + + SetLimited(NET_IPV4, false); + + BOOST_CHECK_EQUAL(IsLocal(addr), false); + BOOST_CHECK_EQUAL(AddLocal(addr, 1000), true); + BOOST_CHECK_EQUAL(IsLocal(addr), true); + + RemoveLocal(addr); + BOOST_CHECK_EQUAL(IsLocal(addr), false); +} + BOOST_AUTO_TEST_CASE(PoissonNextSend) { g_mock_deterministic_tests = true; From 6b607ef101e6c18e315e30fd23e391de7614df38 Mon Sep 17 00:00:00 2001 From: Ben Woosley Date: Wed, 9 Jan 2019 16:41:37 -0800 Subject: [PATCH 20/58] Drop IsLimited in favor of IsReachable These two methods have had the same meaning, but inverted, since 110b62f06992d0fb989153afff2dc3aea62a674f. Having one name for a single concept simplifies the code. --- src/init.cpp | 12 +++++------ src/net.cpp | 32 +++++++++--------------------- src/net.h | 16 ++++++++++----- src/rpc/net.cpp | 2 +- src/test/net_tests.cpp | 45 +++++++++++++++--------------------------- src/torcontrol.cpp | 2 +- 6 files changed, 44 insertions(+), 65 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index b7a0b5f2fd7c1..b14b2e9ce667d 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -390,7 +390,7 @@ static void registerSignalHandler(int signal, void(*handler)(int)) bool static Bind(CConnman& connman, const CService& addr, unsigned int flags) { - if (!(flags & BF_EXPLICIT) && IsLimited(addr)) + if (!(flags & BF_EXPLICIT) && !IsReachable(addr)) return false; std::string strError; if (!connman.BindListenPort(addr, strError, (flags & BF_WHITELIST) != 0)) { @@ -1384,7 +1384,7 @@ bool AppInitMain() for (int n = 0; n < NET_MAX; n++) { enum Network net = (enum Network)n; if (!nets.count(net)) - SetLimited(net); + SetReachable(net, false); } } @@ -1403,7 +1403,7 @@ bool AppInitMain() // -proxy sets a proxy for all outgoing network traffic // -noproxy (or -proxy=0) as well as the empty string can be used to not set a proxy, this is the default std::string proxyArg = gArgs.GetArg("-proxy", ""); - SetLimited(NET_ONION); + SetReachable(NET_ONION, false); if (!proxyArg.empty() && proxyArg != "0") { CService proxyAddr; if (!Lookup(proxyArg, proxyAddr, 9050, fNameLookup)) { @@ -1418,7 +1418,7 @@ bool AppInitMain() SetProxy(NET_IPV6, addrProxy); SetProxy(NET_ONION, addrProxy); SetNameProxy(addrProxy); - SetLimited(NET_ONION, false); // by default, -proxy sets onion as reachable, unless -noonion later + SetReachable(NET_ONION, true); // by default, -proxy sets onion as reachable, unless -noonion later } // -onion can be used to set only a proxy for .onion, or override normal proxy for .onion addresses @@ -1427,7 +1427,7 @@ bool AppInitMain() std::string onionArg = gArgs.GetArg("-onion", ""); if (!onionArg.empty()) { if (onionArg == "0") { // Handle -noonion/-onion=0 - SetLimited(NET_ONION); // set onions as unreachable + SetReachable(NET_ONION, false); } else { CService onionProxy; if (!Lookup(onionArg, onionProxy, 9050, fNameLookup)) { @@ -1437,7 +1437,7 @@ bool AppInitMain() if (!addrOnion.IsValid()) return UIError(strprintf(_("%s Invalid %s address or hostname: '%s'"), "isValid():", "-onion", onionArg)); SetProxy(NET_ONION, addrOnion); - SetLimited(NET_ONION, false); + SetReachable(NET_ONION, true); } } diff --git a/src/net.cpp b/src/net.cpp index 53bfacb4a3dae..16add4d89c652 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -157,7 +157,7 @@ bool IsPeerAddrLocalGood(CNode* pnode) { CService addrLocal = pnode->GetAddrLocal(); return fDiscover && pnode->addr.IsRoutable() && addrLocal.IsRoutable() && - !IsLimited(addrLocal.GetNetwork()); + IsReachable(addrLocal.GetNetwork()); } // pushes our own address to a peer @@ -189,7 +189,7 @@ bool AddLocal(const CService& addr, int nScore) if (!fDiscover && nScore < LOCAL_MANUAL) return false; - if (IsLimited(addr)) + if (!IsReachable(addr)) return false; LogPrintf("AddLocal(%s,%i)\n", addr.ToString(), nScore); @@ -220,24 +220,23 @@ bool RemoveLocal(const CService& addr) return true; } -/** Make a particular network entirely off-limits (no automatic connects to it) */ -void SetLimited(enum Network net, bool fLimited) +void SetReachable(enum Network net, bool reachable) { if (net == NET_UNROUTABLE || net == NET_INTERNAL) return; LOCK(cs_mapLocalHost); - vfLimited[net] = fLimited; + vfLimited[net] = !reachable; } -bool IsLimited(enum Network net) +bool IsReachable(enum Network net) { LOCK(cs_mapLocalHost); - return vfLimited[net]; + return !vfLimited[net]; } -bool IsLimited(const CNetAddr& addr) +bool IsReachable(const CNetAddr& addr) { - return IsLimited(addr.GetNetwork()); + return IsReachable(addr.GetNetwork()); } /** vote for a local address */ @@ -260,19 +259,6 @@ bool IsLocal(const CService& addr) return mapLocalHost.count(addr) > 0; } -/** check whether a given network is one we can probably connect to */ -bool IsReachable(enum Network net) -{ - return !IsLimited(net); -} - -/** check whether a given address is in a network we can probably connect to */ -bool IsReachable(const CNetAddr& addr) -{ - return IsReachable(addr.GetNetwork()); -} - - CNode* CConnman::FindNode(const CNetAddr& ip) { LOCK(cs_vNodes); @@ -1577,7 +1563,7 @@ void CConnman::ThreadOpenConnections() if (nTries > 100) break; - if (IsLimited(addr)) + if (!IsReachable(addr)) continue; // only connect to full nodes diff --git a/src/net.h b/src/net.h index 2ed939e91a557..4044f24e4db27 100644 --- a/src/net.h +++ b/src/net.h @@ -416,17 +416,23 @@ enum { bool IsPeerAddrLocalGood(CNode* pnode); void AdvertiseLocal(CNode* pnode); -void SetLimited(enum Network net, bool fLimited = true); -bool IsLimited(enum Network net); -bool IsLimited(const CNetAddr& addr); + +/** + * Mark a network as reachable or unreachable (no automatic connects to it) + * @note Networks are reachable by default + */ +void SetReachable(enum Network net, bool reachable); +/** @returns true if the network is reachable, false otherwise */ +bool IsReachable(enum Network net); +/** @returns true if the address is in a reachable network, false otherwise */ +bool IsReachable(const CNetAddr& addr); + bool AddLocal(const CService& addr, int nScore = LOCAL_NONE); bool AddLocal(const CNetAddr& addr, int nScore = LOCAL_NONE); bool RemoveLocal(const CService& addr); bool SeenLocal(const CService& addr); bool IsLocal(const CService& addr); bool GetLocal(CService& addr, const CNetAddr* paddrPeer = NULL); -bool IsReachable(enum Network net); -bool IsReachable(const CNetAddr& addr); CAddress GetLocalAddress(const CNetAddr* paddrPeer, ServiceFlags nLocalServices); bool validateMasternodeIP(const std::string& addrStr); // valid, reachable and routable address diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index a375d1da5a502..2479d58a189fa 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -355,7 +355,7 @@ static UniValue GetNetworksInfo() UniValue obj(UniValue::VOBJ); GetProxy(network, proxy); obj.pushKV("name", GetNetworkName(network)); - obj.pushKV("limited", IsLimited(network)); + obj.pushKV("limited", !IsReachable(network)); obj.pushKV("reachable", IsReachable(network)); obj.pushKV("proxy", proxy.IsValid() ? proxy.proxy.ToStringIPPort() : std::string()); obj.pushKV("proxy_randomize_credentials", proxy.randomize_credentials); diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index ccb66adfc3571..3db440dcad16f 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -233,26 +233,21 @@ BOOST_AUTO_TEST_CASE(ipv4_peer_with_ipv6_addrMe_test) BOOST_AUTO_TEST_CASE(LimitedAndReachable_Network) { - SetLimited(NET_IPV4, true); - SetLimited(NET_IPV6, true); - SetLimited(NET_ONION, true); + BOOST_CHECK_EQUAL(IsReachable(NET_IPV4), true); + BOOST_CHECK_EQUAL(IsReachable(NET_IPV6), true); + BOOST_CHECK_EQUAL(IsReachable(NET_ONION), true); - BOOST_CHECK_EQUAL(IsLimited(NET_IPV4), true); - BOOST_CHECK_EQUAL(IsLimited(NET_IPV6), true); - BOOST_CHECK_EQUAL(IsLimited(NET_ONION), true); + SetReachable(NET_IPV4, false); + SetReachable(NET_IPV6, false); + SetReachable(NET_ONION, false); BOOST_CHECK_EQUAL(IsReachable(NET_IPV4), false); BOOST_CHECK_EQUAL(IsReachable(NET_IPV6), false); BOOST_CHECK_EQUAL(IsReachable(NET_ONION), false); - - SetLimited(NET_IPV4, false); - SetLimited(NET_IPV6, false); - SetLimited(NET_ONION, false); - - BOOST_CHECK_EQUAL(IsLimited(NET_IPV4), false); - BOOST_CHECK_EQUAL(IsLimited(NET_IPV6), false); - BOOST_CHECK_EQUAL(IsLimited(NET_ONION), false); + SetReachable(NET_IPV4, true); + SetReachable(NET_IPV6, true); + SetReachable(NET_ONION, true); BOOST_CHECK_EQUAL(IsReachable(NET_IPV4), true); BOOST_CHECK_EQUAL(IsReachable(NET_IPV6), true); @@ -261,19 +256,13 @@ BOOST_AUTO_TEST_CASE(LimitedAndReachable_Network) BOOST_AUTO_TEST_CASE(LimitedAndReachable_NetworkCaseUnroutableAndInternal) { - BOOST_CHECK_EQUAL(IsLimited(NET_UNROUTABLE), false); - BOOST_CHECK_EQUAL(IsLimited(NET_INTERNAL), false); - BOOST_CHECK_EQUAL(IsReachable(NET_UNROUTABLE), true); BOOST_CHECK_EQUAL(IsReachable(NET_INTERNAL), true); - SetLimited(NET_UNROUTABLE, true); - SetLimited(NET_INTERNAL, true); + SetReachable(NET_UNROUTABLE, false); + SetReachable(NET_INTERNAL, false); - BOOST_CHECK_EQUAL(IsLimited(NET_UNROUTABLE), false); // Ignored for both networks - BOOST_CHECK_EQUAL(IsLimited(NET_INTERNAL), false); - - BOOST_CHECK_EQUAL(IsReachable(NET_UNROUTABLE), true); + BOOST_CHECK_EQUAL(IsReachable(NET_UNROUTABLE), true); // Ignored for both networks BOOST_CHECK_EQUAL(IsReachable(NET_INTERNAL), true); } @@ -292,15 +281,13 @@ BOOST_AUTO_TEST_CASE(LimitedAndReachable_CNetAddr) { CNetAddr addr = UtilBuildAddress(0x001, 0x001, 0x001, 0x001); // 1.1.1.1 - SetLimited(NET_IPV4, false); - BOOST_CHECK_EQUAL(IsLimited(addr), false); + SetReachable(NET_IPV4, true); BOOST_CHECK_EQUAL(IsReachable(addr), true); - SetLimited(NET_IPV4, true); - BOOST_CHECK_EQUAL(IsLimited(addr), true); + SetReachable(NET_IPV4, false); BOOST_CHECK_EQUAL(IsReachable(addr), false); - SetLimited(NET_IPV4, false); // have to reset this, because this is stateful. + SetReachable(NET_IPV4, true); // have to reset this, because this is stateful. } @@ -308,7 +295,7 @@ BOOST_AUTO_TEST_CASE(LocalAddress_BasicLifecycle) { CService addr = CService(UtilBuildAddress(0x002, 0x001, 0x001, 0x001), 1000); // 2.1.1.1:1000 - SetLimited(NET_IPV4, false); + SetReachable(NET_IPV4, true); BOOST_CHECK_EQUAL(IsLocal(addr), false); BOOST_CHECK_EQUAL(AddLocal(addr, 1000), true); diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 485057a6b0d65..6bce9b322e685 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -531,7 +531,7 @@ void TorController::auth_cb(TorControlConnection& _conn, const TorControlReply& CService resolved(LookupNumeric("127.0.0.1", 9050)); proxyType addrOnion = proxyType(resolved, true); SetProxy(NET_ONION, addrOnion); - SetLimited(NET_ONION, false); + SetReachable(NET_ONION, true); } // Finally - now create the service From 910d5c470f87029ef4211aa9b7cb590a57292cb6 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Fri, 1 Nov 2019 19:28:15 +0200 Subject: [PATCH 21/58] test: Do not instantiate CAddrDB for static call --- src/test/net_tests.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 3db440dcad16f..b00422a59d45a 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -130,9 +130,8 @@ BOOST_AUTO_TEST_CASE(caddrdb_read) CDataStream ssPeers2 = AddrmanToStream(addrmanUncorrupted); CAddrMan addrman2; - CAddrDB adb; BOOST_CHECK(addrman2.size() == 0); - adb.Read(addrman2, ssPeers2); + BOOST_CHECK(CAddrDB::Read(addrman2, ssPeers2)); BOOST_CHECK(addrman2.size() == 3); } @@ -162,9 +161,8 @@ BOOST_AUTO_TEST_CASE(caddrdb_read_corrupted) CDataStream ssPeers2 = AddrmanToStream(addrmanCorrupted); CAddrMan addrman2; - CAddrDB adb; BOOST_CHECK(addrman2.size() == 0); - adb.Read(addrman2, ssPeers2); + BOOST_CHECK(!CAddrDB::Read(addrman2, ssPeers2)); BOOST_CHECK(addrman2.size() == 0); } From 3337219ca8711fd2248e06e14df48c2aec87acc2 Mon Sep 17 00:00:00 2001 From: furszy Date: Mon, 7 Jun 2021 00:49:46 -0300 Subject: [PATCH 22/58] net: improve encapsulation of CNetAddr. This improvement will help later when we change the type of `CNetAddr::ip` (in the BIP155 implementation). Adaptation of btc@bc74a40a56128f81f11151d5966f53b82f19038c --- src/netaddress.cpp | 10 ++++------ src/netaddress.h | 1 + 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 7c0628cafdcb8..771c88ce94de6 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -666,12 +666,10 @@ bool CService::GetSockAddr(struct sockaddr* paddr, socklen_t *addrlen) const std::vector CService::GetKey() const { - std::vector vKey; - vKey.resize(18); - memcpy(vKey.data(), ip, 16); - vKey[16] = port / 0x100; - vKey[17] = port & 0x0FF; - return vKey; + auto key = GetAddrBytes(); + key.push_back(port / 0x100); // most significant byte of our port + key.push_back(port & 0x0FF); // least significant byte of our port + return key; } std::string CService::ToStringPort() const diff --git a/src/netaddress.h b/src/netaddress.h index 770a7b1ef760e..11fff346c3fb9 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -133,6 +133,7 @@ class CNetAddr uint32_t GetMappedAS(const std::vector &asmap) const; std::vector GetGroup(const std::vector &asmap) const; + std::vector GetAddrBytes() const { return {std::begin(ip), std::end(ip)}; } int GetReachabilityFrom(const CNetAddr* paddrPartner = nullptr) const; CNetAddr(const struct in6_addr& pipv6Addr, const uint32_t scope = 0); From 08ad06d76ecc1aafbca89b0fa16cb0aed71d8a66 Mon Sep 17 00:00:00 2001 From: furszy Date: Mon, 7 Jun 2021 01:41:32 -0300 Subject: [PATCH 23/58] net: change CNetAddr::ip to have flexible size Adaptation of btc@102867c587f5f7954232fb8ed8e85cda78bb4d32 Before this change `CNetAddr::ip` was a fixed-size array of 16 bytes, not being able to store larger addresses (e.g. TORv3) and encoded smaller ones as 16-byte IPv6 addresses. Change its type to `prevector`, so that it can hold larger addresses and do not disguise non-IPv6 addresses as IPv6. So the IPv4 address `1.2.3.4` is now encoded as `01020304` instead of `00000000000000000000FFFF01020304`. Rename `CNetAddr::ip` to `CNetAddr::m_addr` because it is not an "IP" or "IP address" (TOR addresses are not IP addresses). In order to preserve backward compatibility with serialization (where e.g. `1.2.3.4` is serialized as `00000000000000000000FFFF01020304`) introduce `CNetAddr` dedicated legacy serialize/unserialize methods. Adjust `CSubNet` accordingly. Still use `CSubNet::netmask[]` of fixed 16 bytes, but use the first 4 for IPv4 (not the last 4). Only allow subnetting for IPv4 and IPv6. --- src/netaddress.cpp | 383 +++++++++++++++++++++---------------- src/netaddress.h | 151 +++++++++++++-- src/netbase.cpp | 7 +- src/test/net_tests.cpp | 73 +++++++ src/test/netbase_tests.cpp | 9 +- src/utilstrencodings.cpp | 29 +++ src/utilstrencodings.h | 14 ++ 7 files changed, 475 insertions(+), 191 deletions(-) diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 771c88ce94de6..1b67dad75658a 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -10,56 +10,85 @@ #include "util/asmap.h" #include "tinyformat.h" +#include +#include #include +#include +#include -static const unsigned char pchIPv4[12] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff }; -static const unsigned char pchOnionCat[] = {0xFD,0x87,0xD8,0x7E,0xEB,0x43}; +constexpr size_t CNetAddr::V1_SERIALIZATION_SIZE; -// 0xFD + sha256("bitcoin")[0:5] -static const unsigned char g_internal_prefix[] = { 0xFD, 0x6B, 0x88, 0xC0, 0x87, 0x24 }; +/** + * Construct an unspecified IPv6 network address (::/128). + * + * @note This address is considered invalid by CNetAddr::IsValid() + */ +CNetAddr::CNetAddr() {} -CNetAddr::CNetAddr() +void CNetAddr::SetIP(const CNetAddr& ipIn) { - memset(ip, 0, sizeof(ip)); + // Size check. + switch (ipIn.m_net) { + case NET_IPV4: + assert(ipIn.m_addr.size() == ADDR_IPV4_SIZE); + break; + case NET_IPV6: + assert(ipIn.m_addr.size() == ADDR_IPV6_SIZE); + break; + case NET_ONION: + assert(ipIn.m_addr.size() == ADDR_TORV2_SIZE); + break; + case NET_INTERNAL: + assert(ipIn.m_addr.size() == ADDR_INTERNAL_SIZE); + break; + case NET_UNROUTABLE: + case NET_MAX: + assert(false); + } // no default case, so the compiler can warn about missing cases + + m_net = ipIn.m_net; + m_addr = ipIn.m_addr; } -void CNetAddr::SetIP(const CNetAddr& ipIn) +template +inline bool HasPrefix(const T1& obj, const std::array& prefix) { - m_net = ipIn.m_net; - memcpy(ip, ipIn.ip, sizeof(ip)); + return obj.size() >= PREFIX_LEN && + std::equal(std::begin(prefix), std::end(prefix), std::begin(obj)); } -void CNetAddr::SetLegacyIPv6(const uint8_t ipv6[16]) +void CNetAddr::SetLegacyIPv6(Span ipv6) { - if (memcmp(ipv6, pchIPv4, sizeof(pchIPv4)) == 0) { + assert(ipv6.size() == ADDR_IPV6_SIZE); + + size_t skip{0}; + + if (HasPrefix(ipv6, IPV4_IN_IPV6_PREFIX)) { + // IPv4-in-IPv6 m_net = NET_IPV4; - } else if (memcmp(ipv6, pchOnionCat, sizeof(pchOnionCat)) == 0) { + skip = sizeof(IPV4_IN_IPV6_PREFIX); + } else if (HasPrefix(ipv6, TORV2_IN_IPV6_PREFIX)) { + // TORv2-in-IPv6 m_net = NET_ONION; - } else if (memcmp(ipv6, g_internal_prefix, sizeof(g_internal_prefix)) == 0) { + skip = sizeof(TORV2_IN_IPV6_PREFIX); + } else if (HasPrefix(ipv6, INTERNAL_IN_IPV6_PREFIX)) { + // Internal-in-IPv6 m_net = NET_INTERNAL; + skip = sizeof(INTERNAL_IN_IPV6_PREFIX); } else { + // IPv6 m_net = NET_IPV6; } - memcpy(ip, ipv6, 16); -} -void CNetAddr::SetRaw(Network network, const uint8_t* ip_in) -{ - switch(network) - { - case NET_IPV4: - m_net = NET_IPV4; - memcpy(ip, pchIPv4, 12); - memcpy(ip+12, ip_in, 4); - break; - case NET_IPV6: - SetLegacyIPv6(ip_in); - break; - default: - assert(!"invalid network"); - } + m_addr.assign(ipv6.begin() + skip, ipv6.end()); } +/** + * Create an "internal" address that represents a name or FQDN. CAddrMan uses + * these fake addresses to keep track of which DNS seeds were used. + * @returns Whether or not the operation was successful. + * @see NET_INTERNAL, INTERNAL_IN_IPV6_PREFIX, CNetAddr::IsInternal(), CNetAddr::IsRFC4193() + */ bool CNetAddr::SetInternal(const std::string &name) { if (name.empty()) { @@ -68,21 +97,26 @@ bool CNetAddr::SetInternal(const std::string &name) m_net = NET_INTERNAL; unsigned char hash[32] = {}; CSHA256().Write((const unsigned char*)name.data(), name.size()).Finalize(hash); - memcpy(ip, g_internal_prefix, sizeof(g_internal_prefix)); - memcpy(ip + sizeof(g_internal_prefix), hash, sizeof(ip) - sizeof(g_internal_prefix)); + m_addr.assign(hash, hash + ADDR_INTERNAL_SIZE); return true; } +/** + * Parse a TORv2 address and set this object to it. + * + * @returns Whether or not the operation was successful. + * + * @see CNetAddr::IsTor() + */ bool CNetAddr::SetSpecial(const std::string& strName) { if (strName.size()>6 && strName.substr(strName.size() - 6, 6) == ".onion") { std::vector vchAddr = DecodeBase32(strName.substr(0, strName.size() - 6).c_str()); - if (vchAddr.size() != 16-sizeof(pchOnionCat)) + if (vchAddr.size() != ADDR_TORV2_SIZE) { return false; + } m_net = NET_ONION; - memcpy(ip, pchOnionCat, sizeof(pchOnionCat)); - for (unsigned int i=0; i<16-sizeof(pchOnionCat); i++) - ip[i + sizeof(pchOnionCat)] = vchAddr[i]; + m_addr.assign(vchAddr.begin(), vchAddr.end()); return true; } return false; @@ -90,28 +124,23 @@ bool CNetAddr::SetSpecial(const std::string& strName) CNetAddr::CNetAddr(const struct in_addr& ipv4Addr) { - SetRaw(NET_IPV4, (const uint8_t*)&ipv4Addr); + m_net = NET_IPV4; + const uint8_t* ptr = reinterpret_cast(&ipv4Addr); + m_addr.assign(ptr, ptr + ADDR_IPV4_SIZE); } CNetAddr::CNetAddr(const struct in6_addr& ipv6Addr, const uint32_t scope) { - SetRaw(NET_IPV6, (const uint8_t*)&ipv6Addr); + SetLegacyIPv6(Span(reinterpret_cast(&ipv6Addr), sizeof(ipv6Addr))); scopeId = scope; } -unsigned int CNetAddr::GetByte(int n) const -{ - return ip[15-n]; -} - bool CNetAddr::IsBindAny() const { - const int cmplen = IsIPv4() ? 4 : 16; - for (int i = 0; i < cmplen; ++i) { - if (GetByte(i)) return false; + if (!IsIPv4() && !IsIPv6()) { + return false; } - - return true; + return std::all_of(m_addr.begin(), m_addr.end(), [](uint8_t b) { return b == 0; }); } bool CNetAddr::IsIPv4() const { return m_net == NET_IPV4; } @@ -121,88 +150,88 @@ bool CNetAddr::IsIPv6() const { return m_net == NET_IPV6; } bool CNetAddr::IsRFC1918() const { return IsIPv4() && ( - GetByte(3) == 10 || - (GetByte(3) == 192 && GetByte(2) == 168) || - (GetByte(3) == 172 && (GetByte(2) >= 16 && GetByte(2) <= 31))); + m_addr[0] == 10 || + (m_addr[0] == 192 && m_addr[1] == 168) || + (m_addr[0] == 172 && m_addr[1] >= 16 && m_addr[1] <= 31)); } bool CNetAddr::IsRFC2544() const { - return IsIPv4() && GetByte(3) == 198 && (GetByte(2) == 18 || GetByte(2) == 19); + return IsIPv4() && m_addr[0] == 198 && (m_addr[1] == 18 || m_addr[1] == 19); } bool CNetAddr::IsRFC3927() const { - return IsIPv4() && (GetByte(3) == 169 && GetByte(2) == 254); + return IsIPv4() && HasPrefix(m_addr, std::array{169, 254}); } bool CNetAddr::IsRFC6598() const { - return IsIPv4() && GetByte(3) == 100 && GetByte(2) >= 64 && GetByte(2) <= 127; + return IsIPv4() && m_addr[0] == 100 && m_addr[1] >= 64 && m_addr[1] <= 127; } bool CNetAddr::IsRFC5737() const { - return IsIPv4() && ((GetByte(3) == 192 && GetByte(2) == 0 && GetByte(1) == 2) || - (GetByte(3) == 198 && GetByte(2) == 51 && GetByte(1) == 100) || - (GetByte(3) == 203 && GetByte(2) == 0 && GetByte(1) == 113)); + return IsIPv4() && (HasPrefix(m_addr, std::array{192, 0, 2}) || + HasPrefix(m_addr, std::array{198, 51, 100}) || + HasPrefix(m_addr, std::array{203, 0, 113})); } bool CNetAddr::IsRFC3849() const { - return IsIPv6() && GetByte(15) == 0x20 && GetByte(14) == 0x01 && - GetByte(13) == 0x0D && GetByte(12) == 0xB8; + return IsIPv6() && HasPrefix(m_addr, std::array{0x20, 0x01, 0x0D, 0xB8}); } bool CNetAddr::IsRFC3964() const { - return IsIPv6() && GetByte(15) == 0x20 && GetByte(14) == 0x02; + return IsIPv6() && HasPrefix(m_addr, std::array{0x20, 0x02}); } bool CNetAddr::IsRFC6052() const { - static const unsigned char pchRFC6052[] = {0,0x64,0xFF,0x9B,0,0,0,0,0,0,0,0}; - return IsIPv6() && memcmp(ip, pchRFC6052, sizeof(pchRFC6052)) == 0; + return IsIPv6() && + HasPrefix(m_addr, std::array{0x00, 0x64, 0xFF, 0x9B, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}); } bool CNetAddr::IsRFC4380() const { - return IsIPv6() && GetByte(15) == 0x20 && GetByte(14) == 0x01 && GetByte(13) == 0 && - GetByte(12) == 0; + return IsIPv6() && HasPrefix(m_addr, std::array{0x20, 0x01, 0x00, 0x00}); } bool CNetAddr::IsRFC4862() const { - static const unsigned char pchRFC4862[] = {0xFE,0x80,0,0,0,0,0,0}; - return IsIPv6() && memcmp(ip, pchRFC4862, sizeof(pchRFC4862)) == 0; + return IsIPv6() && HasPrefix(m_addr, std::array{0xFE, 0x80, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}); } bool CNetAddr::IsRFC4193() const { - return IsIPv6() && (GetByte(15) & 0xFE) == 0xFC; + return IsIPv6() && (m_addr[0] & 0xFE) == 0xFC; } bool CNetAddr::IsRFC6145() const { - static const unsigned char pchRFC6145[] = {0,0,0,0,0,0,0,0,0xFF,0xFF,0,0}; - return IsIPv6() && memcmp(ip, pchRFC6145, sizeof(pchRFC6145)) == 0; + return IsIPv6() && + HasPrefix(m_addr, std::array{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00}); } bool CNetAddr::IsRFC4843() const { - return IsIPv6() && GetByte(15) == 0x20 && GetByte(14) == 0x01 && - GetByte(13) == 0x00 && (GetByte(12) & 0xF0) == 0x10; + return IsIPv6() && HasPrefix(m_addr, std::array{0x20, 0x01, 0x00}) && + (m_addr[3] & 0xF0) == 0x10; } bool CNetAddr::IsRFC7343() const { - return IsIPv6() && GetByte(15) == 0x20 && GetByte(14) == 0x01 && - GetByte(13) == 0x00 && (GetByte(12) & 0xF0) == 0x20; + return IsIPv6() && HasPrefix(m_addr, std::array{0x20, 0x01, 0x00}) && + (m_addr[3] & 0xF0) == 0x20; } bool CNetAddr::IsHeNet() const { - return (GetByte(15) == 0x20 && GetByte(14) == 0x01 && GetByte(13) == 0x04 && GetByte(12) == 0x70); + return IsIPv6() && HasPrefix(m_addr, std::array{0x20, 0x01, 0x04, 0x70}); } /** @@ -215,14 +244,16 @@ bool CNetAddr::IsTor() const { return m_net == NET_ONION; } bool CNetAddr::IsLocal() const { - // IPv4 loopback - if (IsIPv4() && (GetByte(3) == 127 || GetByte(3) == 0)) - return true; + // IPv4 loopback (127.0.0.0/8 or 0.0.0.0/8) + if (IsIPv4() && (m_addr[0] == 127 || m_addr[0] == 0)) { + return true; + } // IPv6 loopback (::1/128) static const unsigned char pchLocal[16] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1}; - if (IsIPv6() && memcmp(ip, pchLocal, 16) == 0) + if (IsIPv6() && memcmp(m_addr.data(), pchLocal, sizeof(pchLocal)) == 0) { return true; + } return false; } @@ -235,13 +266,16 @@ bool CNetAddr::IsValid() const // header20 vectorlen3 addr26 addr26 addr26 header20 vectorlen3 addr26 addr26 addr26... // so if the first length field is garbled, it reads the second batch // of addr misaligned by 3 bytes. - if (IsIPv6() && memcmp(ip, pchIPv4+3, sizeof(pchIPv4)-3) == 0) + if (IsIPv6() && memcmp(m_addr.data(), IPV4_IN_IPV6_PREFIX.data() + 3, + sizeof(IPV4_IN_IPV6_PREFIX) - 3) == 0) { return false; + } // unspecified IPv6 address (::/128) unsigned char ipNone6[16] = {}; - if (IsIPv6() && memcmp(ip, ipNone6, 16) == 0) + if (IsIPv6() && memcmp(m_addr.data(), ipNone6, sizeof(ipNone6)) == 0) { return false; + } // documentation IPv6 address if (IsRFC3849()) @@ -250,17 +284,11 @@ bool CNetAddr::IsValid() const if (IsInternal()) return false; - if (IsIPv4()) - { - // INADDR_NONE - uint32_t ipNone = INADDR_NONE; - if (memcmp(ip+12, &ipNone, 4) == 0) - return false; - - // 0 - ipNone = 0; - if (memcmp(ip+12, &ipNone, 4) == 0) + if (IsIPv4()) { + const uint32_t addr = ReadBE32(m_addr.data()); + if (addr == INADDR_ANY || addr == INADDR_NONE) { return false; + } } return true; @@ -271,6 +299,11 @@ bool CNetAddr::IsRoutable() const return IsValid() && !(IsRFC1918() || IsRFC2544() || IsRFC3927() || IsRFC4862() || IsRFC6598() || IsRFC5737() || (IsRFC4193() && !IsTor()) || IsRFC4843() || IsRFC7343() || IsLocal() || IsInternal()); } +/** + * @returns Whether or not this is a dummy address that represents a name. + * + * @see CNetAddr::SetInternal(const std::string &) + */ bool CNetAddr::IsInternal() const { return m_net == NET_INTERNAL; @@ -290,9 +323,9 @@ enum Network CNetAddr::GetNetwork() const std::string CNetAddr::ToStringIP() const { if (IsTor()) - return EncodeBase32(&ip[6], 10) + ".onion"; + return EncodeBase32(m_addr.data(), m_addr.size()) + ".onion"; if (IsInternal()) - return EncodeBase32(ip + sizeof(g_internal_prefix), sizeof(ip) - sizeof(g_internal_prefix)) + ".internal"; + return EncodeBase32(m_addr.data(), m_addr.size()) + ".internal"; CService serv(*this, 0); struct sockaddr_storage sockaddr; socklen_t socklen = sizeof(sockaddr); @@ -302,13 +335,13 @@ std::string CNetAddr::ToStringIP() const return std::string(name); } if (IsIPv4()) - return strprintf("%u.%u.%u.%u", GetByte(3), GetByte(2), GetByte(1), GetByte(0)); - else - return strprintf("%x:%x:%x:%x:%x:%x:%x:%x", - GetByte(15) << 8 | GetByte(14), GetByte(13) << 8 | GetByte(12), - GetByte(11) << 8 | GetByte(10), GetByte(9) << 8 | GetByte(8), - GetByte(7) << 8 | GetByte(6), GetByte(5) << 8 | GetByte(4), - GetByte(3) << 8 | GetByte(2), GetByte(1) << 8 | GetByte(0)); + return strprintf("%u.%u.%u.%u", m_addr[0], m_addr[1], m_addr[2], m_addr[3]); + assert(IsIPv6()); + return strprintf("%x:%x:%x:%x:%x:%x:%x:%x", + m_addr[0] << 8 | m_addr[1], m_addr[2] << 8 | m_addr[3], + m_addr[4] << 8 | m_addr[5], m_addr[6] << 8 | m_addr[7], + m_addr[8] << 8 | m_addr[9], m_addr[10] << 8 | m_addr[11], + m_addr[12] << 8 | m_addr[13], m_addr[14] << 8 | m_addr[15]); } std::string CNetAddr::ToString() const @@ -318,24 +351,25 @@ std::string CNetAddr::ToString() const bool operator==(const CNetAddr& a, const CNetAddr& b) { - return a.m_net == b.m_net && memcmp(a.ip, b.ip, 16) == 0; + return a.m_net == b.m_net && a.m_addr == b.m_addr; } bool operator!=(const CNetAddr& a, const CNetAddr& b) { - return a.m_net != b.m_net || (memcmp(a.ip, b.ip, 16) != 0); + return a.m_net != b.m_net || a.m_addr != b.m_addr; } bool operator<(const CNetAddr& a, const CNetAddr& b) { - return a.m_net < b.m_net || (a.m_net == b.m_net && memcmp(a.ip, b.ip, 16) < 0); + return std::tie(a.m_net, a.m_addr) < std::tie(b.m_net, b.m_addr); } bool CNetAddr::GetInAddr(struct in_addr* pipv4Addr) const { if (!IsIPv4()) return false; - memcpy(pipv4Addr, ip+12, 4); + assert(sizeof(*pipv4Addr) == m_addr.size()); + memcpy(pipv4Addr, m_addr.data(), m_addr.size()); return true; } @@ -354,7 +388,8 @@ bool CNetAddr::GetIn6Addr(struct in6_addr* pipv6Addr) const if (!IsIPv6()) { return false; } - memcpy(pipv6Addr, ip, 16); + assert(sizeof(*pipv6Addr) == m_addr.size()); + memcpy(pipv6Addr, m_addr.data(), m_addr.size()); return true; } @@ -365,15 +400,17 @@ bool CNetAddr::HasLinkedIPv4() const uint32_t CNetAddr::GetLinkedIPv4() const { - if (IsIPv4() || IsRFC6145() || IsRFC6052()) { - // IPv4, mapped IPv4, SIIT translated IPv4: the IPv4 address is the last 4 bytes of the address - return ReadBE32(ip + 12); + if (IsIPv4()) { + return ReadBE32(m_addr.data()); + } else if (IsRFC6052() || IsRFC6145()) { + // mapped IPv4, SIIT translated IPv4: the IPv4 address is the last 4 bytes of the address + return ReadBE32(MakeSpan(m_addr).last(ADDR_IPV4_SIZE).data()); } else if (IsRFC3964()) { // 6to4 tunneled IPv4: the IPv4 address is in bytes 2-6 - return ReadBE32(ip + 2); + return ReadBE32(MakeSpan(m_addr).subspan(2, ADDR_IPV4_SIZE).data()); } else if (IsRFC4380()) { // Teredo tunneled IPv4: the IPv4 address is in the last 4 bytes of the address, but bitflipped - return ~ReadBE32(ip + 12); + return ~ReadBE32(MakeSpan(m_addr).last(ADDR_IPV4_SIZE).data()); } assert(false); } @@ -402,10 +439,10 @@ uint32_t CNetAddr::GetMappedAS(const std::vector &asmap) const { } std::vector ip_bits(128); if (HasLinkedIPv4()) { - // For lookup, treat as if it was just an IPv4 address (pchIPv4 prefix + IPv4 bits) + // For lookup, treat as if it was just an IPv4 address (IPV4_IN_IPV6_PREFIX + IPv4 bits) for (int8_t byte_i = 0; byte_i < 12; ++byte_i) { for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) { - ip_bits[byte_i * 8 + bit_i] = (pchIPv4[byte_i] >> (7 - bit_i)) & 1; + ip_bits[byte_i * 8 + bit_i] = (IPV4_IN_IPV6_PREFIX[byte_i] >> (7 - bit_i)) & 1; } } uint32_t ipv4 = GetLinkedIPv4(); @@ -414,8 +451,9 @@ uint32_t CNetAddr::GetMappedAS(const std::vector &asmap) const { } } else { // Use all 128 bits of the IPv6 address otherwise + assert(IsIPv6()); for (int8_t byte_i = 0; byte_i < 16; ++byte_i) { - uint8_t cur_byte = GetByte(15 - byte_i); + uint8_t cur_byte = m_addr[byte_i]; for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) { ip_bits[byte_i * 8 + bit_i] = (cur_byte >> (7 - bit_i)) & 1; } @@ -451,19 +489,15 @@ std::vector CNetAddr::GetGroup(const std::vector &asmap) co } vchRet.push_back(net_class); - int nStartByte = 0; - int nBits = 16; + int nBits{0}; if (IsLocal()) { // all local addresses belong to the same group - nBits = 0; } else if (IsInternal()) { // all internal-usage addresses get their own group - nStartByte = sizeof(g_internal_prefix); - nBits = (sizeof(ip) - sizeof(g_internal_prefix)) * 8; + nBits = ADDR_INTERNAL_SIZE * 8; } else if (!IsRoutable()) { // all other unroutable addresses belong to the same group - nBits = 0; } else if (HasLinkedIPv4()) { // IPv4 addresses (and mapped IPv4 addresses) use /16 groups uint32_t ipv4 = GetLinkedIPv4(); @@ -471,7 +505,6 @@ std::vector CNetAddr::GetGroup(const std::vector &asmap) co vchRet.push_back((ipv4 >> 16) & 0xFF); return vchRet; } else if (IsTor()) { - nStartByte = 6; nBits = 4; } else if (IsHeNet()) { // for he.net, use /36 groups @@ -481,22 +514,29 @@ std::vector CNetAddr::GetGroup(const std::vector &asmap) co nBits = 32; } - // push our ip onto vchRet byte by byte... - while (nBits >= 8) - { - vchRet.push_back(GetByte(15 - nStartByte)); - nStartByte++; - nBits -= 8; + // Push our address onto vchRet. + const size_t num_bytes = nBits / 8; + vchRet.insert(vchRet.end(), m_addr.begin(), m_addr.begin() + num_bytes); + nBits %= 8; + // ...for the last byte, push nBits and for the rest of the byte push 1's + if (nBits > 0) { + assert(num_bytes < m_addr.size()); + vchRet.push_back(m_addr[num_bytes] | ((1 << (8 - nBits)) - 1)); } - if (nBits > 0) - vchRet.push_back(GetByte(15 - nStartByte) | ((1 << (8 - nBits)) - 1)); return vchRet; } +std::vector CNetAddr::GetAddrBytes() const +{ + uint8_t serialized[V1_SERIALIZATION_SIZE]; + SerializeV1Array(serialized); + return {std::begin(serialized), std::end(serialized)}; +} + uint64_t CNetAddr::GetHash() const { - uint256 hash = Hash(&ip[0], &ip[16]); + uint256 hash = Hash(m_addr.begin(), m_addr.end()); uint64_t nRet; memcpy(&nRet, &hash, sizeof(nRet)); return nRet; @@ -697,29 +737,25 @@ CSubNet::CSubNet(): memset(netmask, 0, sizeof(netmask)); } -CSubNet::CSubNet(const CNetAddr &addr, int32_t mask) +CSubNet::CSubNet(const CNetAddr& addr, uint8_t mask) : CSubNet() { - valid = true; + valid = (addr.IsIPv4() && mask <= ADDR_IPV4_SIZE * 8) || + (addr.IsIPv6() && mask <= ADDR_IPV6_SIZE * 8); + if (!valid) { + return; + } + + assert(mask <= sizeof(netmask) * 8); + network = addr; - // Default to /32 (IPv4) or /128 (IPv6), i.e. match single address - memset(netmask, 255, sizeof(netmask)); - - // IPv4 addresses start at offset 12, and first 12 bytes must match, so just offset n - const int astartofs = network.IsIPv4() ? 12 : 0; - - int32_t n = mask; - if(n >= 0 && n <= (128 - astartofs*8)) // Only valid if in range of bits of address - { - n += astartofs*8; - // Clear bits [n..127] - for (; n < 128; ++n) - netmask[n>>3] &= ~(1<<(7-(n&7))); - } else - valid = false; - // Normalize network according to netmask - for(int x=0; x<16; ++x) - network.ip[x] &= netmask[x]; + uint8_t n = mask; + for (size_t i = 0; i < network.m_addr.size(); ++i) { + const uint8_t bits = n < 8 ? n : 8; + netmask[i] = (uint8_t)((uint8_t)0xFF << (8 - bits)); // Set first bits. + network.m_addr[i] &= netmask[i]; // Normalize network according to netmask. + n -= bits; + } } /** @@ -742,13 +778,16 @@ static inline int NetmaskBits(uint8_t x) } } -CSubNet::CSubNet(const CNetAddr &addr, const CNetAddr &mask) +CSubNet::CSubNet(const CNetAddr& addr, const CNetAddr& mask) : CSubNet() { - valid = true; + valid = (addr.IsIPv4() || addr.IsIPv6()) && addr.m_net == mask.m_net; + if (!valid) { + return; + } // Check if `mask` contains 1-bits after 0-bits (which is an invalid netmask). bool zeros_found = false; - for (size_t i = mask.IsIPv4() ? 12 : 0; i < sizeof(mask.ip); ++i) { - const int num_bits = NetmaskBits(mask.ip[i]); + for (auto b : mask.m_addr) { + const int num_bits = NetmaskBits(b); if (num_bits == -1 || (zeros_found && num_bits != 0)) { valid = false; return; @@ -757,25 +796,30 @@ CSubNet::CSubNet(const CNetAddr &addr, const CNetAddr &mask) zeros_found = true; } } - network = addr; - // Default to /32 (IPv4) or /128 (IPv6), i.e. match single address - memset(netmask, 255, sizeof(netmask)); - // IPv4 addresses start at offset 12, and first 12 bytes must match, so just offset n - const int astartofs = network.IsIPv4() ? 12 : 0; + assert(mask.m_addr.size() <= sizeof(netmask)); - for(int x=astartofs; x<16; ++x) - netmask[x] = mask.ip[x]; + memcpy(netmask, mask.m_addr.data(), mask.m_addr.size()); + + network = addr; // Normalize network according to netmask - for(int x=0; x<16; ++x) - network.ip[x] &= netmask[x]; + for (size_t x = 0; x < network.m_addr.size(); ++x) { + network.m_addr[x] &= netmask[x]; + } } -CSubNet::CSubNet(const CNetAddr &addr): - valid(addr.IsValid()) +CSubNet::CSubNet(const CNetAddr& addr) : CSubNet() { - memset(netmask, 255, sizeof(netmask)); + valid = addr.IsIPv4() || addr.IsIPv6(); + if (!valid) { + return; + } + + assert(addr.m_addr.size() <= sizeof(netmask)); + + memset(netmask, 0xFF, addr.m_addr.size()); + network = addr; } @@ -783,17 +827,22 @@ bool CSubNet::Match(const CNetAddr &addr) const { if (!valid || !addr.IsValid() || network.m_net != addr.m_net) return false; - for(int x=0; x<16; ++x) - if ((addr.ip[x] & netmask[x]) != network.ip[x]) + assert(network.m_addr.size() == addr.m_addr.size()); + for (size_t x = 0; x < addr.m_addr.size(); ++x) { + if ((addr.m_addr[x] & netmask[x]) != network.m_addr[x]) { return false; + } + } return true; } std::string CSubNet::ToString() const { + assert(network.m_addr.size() <= sizeof(netmask)); + uint8_t cidr = 0; - for (size_t i = network.IsIPv4() ? 12 : 0; i < sizeof(netmask); ++i) { + for (size_t i = 0; i < network.m_addr.size(); ++i) { if (netmask[i] == 0x00) { break; } diff --git a/src/netaddress.h b/src/netaddress.h index 11fff346c3fb9..6820c1cb053be 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -13,7 +13,9 @@ #include "compat.h" #include "serialize.h" #include "span.h" +#include "prevector.h" +#include #include #include #include @@ -41,28 +43,66 @@ enum Network /// TORv2 NET_ONION, - /// A set of dummy addresses that map a name to an IPv6 address. These - /// addresses belong to RFC4193's fc00::/7 subnet (unique-local addresses). - /// We use them to map a string or FQDN to an IPv6 address in CAddrMan to - /// keep track of which DNS seeds were used. + /// A set of addresses that represent the hash of a string or FQDN. We use + /// them in CAddrMan to keep track of which DNS seeds were used. NET_INTERNAL, /// Dummy value to indicate the number of NET_* constants. NET_MAX, }; +/// Prefix of an IPv6 address when it contains an embedded IPv4 address. +/// Used when (un)serializing addresses in ADDRv1 format (pre-BIP155). +static const std::array IPV4_IN_IPV6_PREFIX{ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF +}; + +/// Prefix of an IPv6 address when it contains an embedded TORv2 address. +/// Used when (un)serializing addresses in ADDRv1 format (pre-BIP155). +/// Such dummy IPv6 addresses are guaranteed to not be publicly routable as they +/// fall under RFC4193's fc00::/7 subnet allocated to unique-local addresses. +static const std::array TORV2_IN_IPV6_PREFIX{ + 0xFD, 0x87, 0xD8, 0x7E, 0xEB, 0x43 +}; + +/// Prefix of an IPv6 address when it contains an embedded "internal" address. +/// Used when (un)serializing addresses in ADDRv1 format (pre-BIP155). +/// The prefix comes from 0xFD + SHA256("bitcoin")[0:5]. +/// Such dummy IPv6 addresses are guaranteed to not be publicly routable as they +/// fall under RFC4193's fc00::/7 subnet allocated to unique-local addresses. +static const std::array INTERNAL_IN_IPV6_PREFIX{ + 0xFD, 0x6B, 0x88, 0xC0, 0x87, 0x24 // 0xFD + sha256("bitcoin")[0:5]. +}; + +/// Size of IPv4 address (in bytes). +static constexpr size_t ADDR_IPV4_SIZE = 4; + +/// Size of IPv6 address (in bytes). +static constexpr size_t ADDR_IPV6_SIZE = 16; + +/// Size of TORv2 address (in bytes). +static constexpr size_t ADDR_TORV2_SIZE = 10; + +/// Size of "internal" (NET_INTERNAL) address (in bytes). +static constexpr size_t ADDR_INTERNAL_SIZE = 10; + /** * Network address. */ class CNetAddr { protected: + /** + * Raw representation of the network address. + * In network byte order (big endian) for IPv4 and IPv6. + */ + prevector m_addr{ADDR_IPV6_SIZE, 0x0}; + /** * Network to which this address belongs. */ Network m_net{NET_IPV6}; - unsigned char ip[16]; // in network byte order uint32_t scopeId{0}; // for scoped/link-local ipv6 addresses public: @@ -76,13 +116,7 @@ class CNetAddr * (e.g. IPv4) disguised as IPv6. This encoding is used in the legacy * `addr` encoding. */ - void SetLegacyIPv6(const uint8_t ipv6[16]); - - /** - * Set raw IPv4 or IPv6 address (in network byte order) - * @note Only NET_IPV4 and NET_IPV6 are allowed for network. - */ - void SetRaw(Network network, const uint8_t* data); + void SetLegacyIPv6(Span ipv6); /** * Transform an arbitrary string into a non-routable ipv6 address. @@ -117,7 +151,6 @@ class CNetAddr enum Network GetNetwork() const; std::string ToString() const; std::string ToStringIP() const; - unsigned int GetByte(int n) const; uint64_t GetHash() const; bool GetInAddr(struct in_addr* pipv4Addr) const; uint32_t GetNetClass() const; @@ -133,7 +166,7 @@ class CNetAddr uint32_t GetMappedAS(const std::vector &asmap) const; std::vector GetGroup(const std::vector &asmap) const; - std::vector GetAddrBytes() const { return {std::begin(ip), std::end(ip)}; } + std::vector GetAddrBytes() const; int GetReachabilityFrom(const CNetAddr* paddrPartner = nullptr) const; CNetAddr(const struct in6_addr& pipv6Addr, const uint32_t scope = 0); @@ -149,7 +182,7 @@ class CNetAddr template void Serialize(Stream& s) const { - s << ip; + SerializeV1Stream(s); } /** @@ -158,14 +191,92 @@ class CNetAddr template void Unserialize(Stream& s) { - unsigned char ip_temp[sizeof(ip)]; - s >> ip_temp; + UnserializeV1Stream(s); + } + +friend class CSubNet; + +private: + /** + * Size of CNetAddr when serialized as ADDRv1 (pre-BIP155) (in bytes). + */ + static constexpr size_t V1_SERIALIZATION_SIZE = ADDR_IPV6_SIZE; + + /** + * Serialize in pre-ADDRv2/BIP155 format to an array. + * Some addresses (e.g. TORv3) cannot be serialized in pre-BIP155 format. + */ + void SerializeV1Array(uint8_t (&arr)[V1_SERIALIZATION_SIZE]) const + { + size_t prefix_size; + + switch (m_net) { + case NET_IPV6: + assert(m_addr.size() == sizeof(arr)); + memcpy(arr, m_addr.data(), m_addr.size()); + return; + case NET_IPV4: + prefix_size = sizeof(IPV4_IN_IPV6_PREFIX); + assert(prefix_size + m_addr.size() == sizeof(arr)); + memcpy(arr, IPV4_IN_IPV6_PREFIX.data(), prefix_size); + memcpy(arr + prefix_size, m_addr.data(), m_addr.size()); + return; + case NET_ONION: + prefix_size = sizeof(TORV2_IN_IPV6_PREFIX); + assert(prefix_size + m_addr.size() == sizeof(arr)); + memcpy(arr, TORV2_IN_IPV6_PREFIX.data(), prefix_size); + memcpy(arr + prefix_size, m_addr.data(), m_addr.size()); + return; + case NET_INTERNAL: + prefix_size = sizeof(INTERNAL_IN_IPV6_PREFIX); + assert(prefix_size + m_addr.size() == sizeof(arr)); + memcpy(arr, INTERNAL_IN_IPV6_PREFIX.data(), prefix_size); + memcpy(arr + prefix_size, m_addr.data(), m_addr.size()); + return; + case NET_UNROUTABLE: + case NET_MAX: + assert(false); + } // no default case, so the compiler can warn about missing cases + + assert(false); + } + + /** + * Serialize in pre-ADDRv2/BIP155 format to a stream. + * Some addresses (e.g. TORv3) cannot be serialized in pre-BIP155 format. + */ + template + void SerializeV1Stream(Stream& s) const + { + uint8_t serialized[V1_SERIALIZATION_SIZE]; + + SerializeV1Array(serialized); + + s << serialized; + } + + /** + * Unserialize from a pre-ADDRv2/BIP155 format from an array. + */ + void UnserializeV1Array(uint8_t (&arr)[V1_SERIALIZATION_SIZE]) + { // Use SetLegacyIPv6() so that m_net is set correctly. For example // ::FFFF:0102:0304 should be set as m_net=NET_IPV4 (1.2.3.4). - SetLegacyIPv6(ip_temp); + SetLegacyIPv6(arr); } - friend class CSubNet; + /** + * Unserialize from a pre-ADDRv2/BIP155 format from a stream. + */ + template + void UnserializeV1Stream(Stream& s) + { + uint8_t serialized[V1_SERIALIZATION_SIZE]; + + s >> serialized; + + UnserializeV1Array(serialized); + } }; class CSubNet @@ -180,7 +291,7 @@ class CSubNet public: CSubNet(); - CSubNet(const CNetAddr& addr, int32_t mask); + CSubNet(const CNetAddr& addr, uint8_t mask); CSubNet(const CNetAddr& addr, const CNetAddr& mask); //constructor for single ip subnet (/32 or /128) diff --git a/src/netbase.cpp b/src/netbase.cpp index 75e8cb72d7a8c..9004540464aca 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -14,6 +14,7 @@ #include #include +#include #ifndef WIN32 #include @@ -695,9 +696,9 @@ bool LookupSubNet(const std::string& strSubnet, CSubNet& ret) CNetAddr network = vIP[0]; if (slash != strSubnet.npos) { std::string strNetmask = strSubnet.substr(slash + 1); - int32_t n; - // IPv4 addresses start at offset 12, and first 12 bytes must match, so just offset n - if (ParseInt32(strNetmask, &n)) { // If valid number, assume /24 syntax + uint8_t n; + if (ParseUInt8(strNetmask, &n)) { + // If valid number, assume CIDR variable-length subnet masking ret = CSubNet(network, n); return ret.IsValid(); } else // If not a valid number, try full netmask syntax diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index b00422a59d45a..c8e8fe1675fea 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -9,6 +9,7 @@ #include "netbase.h" #include "serialize.h" #include "streams.h" +#include "version.h" #include @@ -190,6 +191,78 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) BOOST_CHECK(pnode2->fFeeler == false); } +BOOST_AUTO_TEST_CASE(cnetaddr_basic) +{ + CNetAddr addr; + + // IPv4, INADDR_ANY + BOOST_REQUIRE(LookupHost("0.0.0.0", addr, false)); + BOOST_REQUIRE(!addr.IsValid()); + BOOST_REQUIRE(addr.IsIPv4()); + + BOOST_CHECK(addr.IsBindAny()); + BOOST_CHECK_EQUAL(addr.ToString(), "0.0.0.0"); + + // IPv4, INADDR_NONE + BOOST_REQUIRE(LookupHost("255.255.255.255", addr, false)); + BOOST_REQUIRE(!addr.IsValid()); + BOOST_REQUIRE(addr.IsIPv4()); + + BOOST_CHECK(!addr.IsBindAny()); + BOOST_CHECK_EQUAL(addr.ToString(), "255.255.255.255"); + + // IPv4, casual + BOOST_REQUIRE(LookupHost("12.34.56.78", addr, false)); + BOOST_REQUIRE(addr.IsValid()); + BOOST_REQUIRE(addr.IsIPv4()); + + BOOST_CHECK(!addr.IsBindAny()); + BOOST_CHECK_EQUAL(addr.ToString(), "12.34.56.78"); + + // IPv6, in6addr_any + BOOST_REQUIRE(LookupHost("::", addr, false)); + BOOST_REQUIRE(!addr.IsValid()); + BOOST_REQUIRE(addr.IsIPv6()); + + BOOST_CHECK(addr.IsBindAny()); + BOOST_CHECK_EQUAL(addr.ToString(), "::"); + + // IPv6, casual + BOOST_REQUIRE(LookupHost("1122:3344:5566:7788:9900:aabb:ccdd:eeff", addr, false)); + BOOST_REQUIRE(addr.IsValid()); + BOOST_REQUIRE(addr.IsIPv6()); + + BOOST_CHECK(!addr.IsBindAny()); + BOOST_CHECK_EQUAL(addr.ToString(), "1122:3344:5566:7788:9900:aabb:ccdd:eeff"); + + // TORv2 + addr.SetSpecial("6hzph5hv6337r6p2.onion"); + BOOST_REQUIRE(addr.IsValid()); + BOOST_REQUIRE(addr.IsTor()); + + BOOST_CHECK(!addr.IsBindAny()); + BOOST_CHECK_EQUAL(addr.ToString(), "6hzph5hv6337r6p2.onion"); + + // Internal + addr.SetInternal("esffpp"); + BOOST_REQUIRE(!addr.IsValid()); // "internal" is considered invalid + BOOST_REQUIRE(addr.IsInternal()); + + BOOST_CHECK(!addr.IsBindAny()); + BOOST_CHECK_EQUAL(addr.ToString(), "esffpvrt3wpeaygy.internal"); +} + +BOOST_AUTO_TEST_CASE(cnetaddr_serialize) +{ + CNetAddr addr; + CDataStream s(SER_NETWORK, PROTOCOL_VERSION); + + addr.SetInternal("a"); + s << addr; + BOOST_CHECK_EQUAL(HexStr(s), "fd6b88c08724ca978112ca1bbdcafac2"); + s.clear(); +} + // prior to PR #14728, this test triggers an undefined behavior BOOST_AUTO_TEST_CASE(ipv4_peer_with_ipv6_addrMe_test) { diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 9c12765fc151c..167beaf9df507 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -182,6 +182,7 @@ BOOST_AUTO_TEST_CASE(subnet_test) BOOST_CHECK(!ResolveSubNet("1.2.3.0/-1").IsValid()); BOOST_CHECK(ResolveSubNet("1.2.3.0/32").IsValid()); BOOST_CHECK(!ResolveSubNet("1.2.3.0/33").IsValid()); + BOOST_CHECK(!ResolveSubNet("1.2.3.0/300").IsValid()); BOOST_CHECK(ResolveSubNet("1:2:3:4:5:6:7:8/0").IsValid()); BOOST_CHECK(ResolveSubNet("1:2:3:4:5:6:7:8/33").IsValid()); BOOST_CHECK(!ResolveSubNet("1:2:3:4:5:6:7:8/-1").IsValid()); @@ -213,6 +214,11 @@ BOOST_AUTO_TEST_CASE(subnet_test) BOOST_CHECK(CSubNet(ResolveIP("1:2:3:4:5:6:7:8")).Match(ResolveIP("1:2:3:4:5:6:7:8"))); BOOST_CHECK(!CSubNet(ResolveIP("1:2:3:4:5:6:7:8")).Match(ResolveIP("1:2:3:4:5:6:7:9"))); BOOST_CHECK(CSubNet(ResolveIP("1:2:3:4:5:6:7:8")).ToString() == "1:2:3:4:5:6:7:8/128"); + // IPv4 address with IPv6 netmask or the other way around. + BOOST_CHECK(!CSubNet(ResolveIP("1.1.1.1"), ResolveIP("ffff::")).IsValid()); + BOOST_CHECK(!CSubNet(ResolveIP("::1"), ResolveIP("255.0.0.0")).IsValid()); + // Can't subnet TOR (or any other non-IPv4 and non-IPv6 network). + BOOST_CHECK(!CSubNet(ResolveIP("5wyqrzbvrdsumnok.onion"), ResolveIP("255.0.0.0")).IsValid()); subnet = ResolveSubNet("1.2.3.4/255.255.255.255"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.3.4/32"); @@ -372,7 +378,8 @@ BOOST_AUTO_TEST_CASE(netbase_dont_resolve_strings_with_embedded_nul_characters) BOOST_CHECK(!LookupSubNet(std::string("1.2.3.0/24\0", 11), ret)); BOOST_CHECK(!LookupSubNet(std::string("1.2.3.0/24\0example.com", 22), ret)); BOOST_CHECK(!LookupSubNet(std::string("1.2.3.0/24\0example.com\0", 23), ret)); - BOOST_CHECK(LookupSubNet(std::string("5wyqrzbvrdsumnok.onion", 22), ret)); + // We only do subnetting for IPv4 and IPv6 + BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion", 22), ret)); BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0", 23), ret)); BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0example.com", 34), ret)); BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0example.com\0", 35), ret)); diff --git a/src/utilstrencodings.cpp b/src/utilstrencodings.cpp index 50ff8b6abe0ec..7547af04b472f 100644 --- a/src/utilstrencodings.cpp +++ b/src/utilstrencodings.cpp @@ -303,6 +303,35 @@ bool ParseInt64(const std::string& str, int64_t *out) n <= std::numeric_limits::max(); } +bool ParseUInt8(const std::string& str, uint8_t *out) +{ + uint32_t u32; + if (!ParseUInt32(str, &u32) || u32 > std::numeric_limits::max()) { + return false; + } + if (out != nullptr) { + *out = static_cast(u32); + } + return true; +} + +bool ParseUInt32(const std::string& str, uint32_t *out) +{ + if (!ParsePrechecks(str)) + return false; + if (str.size() >= 1 && str[0] == '-') // Reject negative values, unfortunately strtoul accepts these by default if they fit in the range + return false; + char *endp = nullptr; + errno = 0; // strtoul will not set errno if valid + unsigned long int n = strtoul(str.c_str(), &endp, 10); + if(out) *out = (uint32_t)n; + // Note that strtoul returns a *unsigned long int*, so even if it doesn't report an over/underflow + // we still have to check that the returned value is within the range of an *uint32_t*. On 64-bit + // platforms the size of these types may be different. + return endp && *endp == 0 && !errno && + n <= std::numeric_limits::max(); +} + bool ParseDouble(const std::string& str, double *out) { if (!ParsePrechecks(str)) diff --git a/src/utilstrencodings.h b/src/utilstrencodings.h index 88ce022f31544..9ca81339aa611 100644 --- a/src/utilstrencodings.h +++ b/src/utilstrencodings.h @@ -92,6 +92,20 @@ bool ParseInt32(const std::string& str, int32_t *out); */ bool ParseInt64(const std::string& str, int64_t *out); +/** + * Convert decimal string to unsigned 8-bit integer with strict parse error feedback. + * @returns true if the entire string could be parsed as valid integer, + * false if not the entire string could be parsed or when overflow or underflow occurred. + */ +bool ParseUInt8(const std::string& str, uint8_t *out); + +/** + * Convert decimal string to unsigned 32-bit integer with strict parse error feedback. + * @returns true if the entire string could be parsed as valid integer, + * false if not the entire string could be parsed or when overflow or underflow occurred. + */ +bool ParseUInt32(const std::string& str, uint32_t *out); + /** * Convert string to double with strict parse error feedback. * @returns true if the entire string could be parsed as valid double, From 45222e69528a433504c22127a810d97c0a247c9c Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 30 Aug 2020 12:54:45 -0700 Subject: [PATCH 24/58] Implement keccak-f[1600] and SHA3-256 --- CMakeLists.txt | 1 + src/Makefile.am | 2 + src/crypto/sha3.cpp | 128 ++++++++++++++++++++++++++++++++++++++ src/crypto/sha3.h | 41 ++++++++++++ src/test/crypto_tests.cpp | 107 +++++++++++++++++++++++++++++++ 5 files changed, 279 insertions(+) create mode 100644 src/crypto/sha3.cpp create mode 100644 src/crypto/sha3.h diff --git a/CMakeLists.txt b/CMakeLists.txt index e2426055b91cf..de1cd0ec9e555 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -291,6 +291,7 @@ set(BITCOIN_CRYPTO_SOURCES ./src/crypto/sha1.cpp ./src/crypto/sha256.cpp ./src/crypto/sha512.cpp + ./src/crypto/sha3.cpp ./src/crypto/chacha20.cpp ./src/crypto/hmac_sha256.cpp ./src/crypto/rfc6979_hmac_sha256.cpp diff --git a/src/Makefile.am b/src/Makefile.am index 80f8410cc30a0..bcfef026a9e6b 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -448,6 +448,8 @@ crypto_libbitcoin_crypto_a_SOURCES = \ crypto/skein.c \ crypto/common.h \ crypto/sha256.h \ + crypto/sha3.h \ + crypto/sha3.cpp \ crypto/sha512.h \ crypto/hmac_sha256.h \ crypto/rfc6979_hmac_sha256.h \ diff --git a/src/crypto/sha3.cpp b/src/crypto/sha3.cpp new file mode 100644 index 0000000000000..5f9ce451461c4 --- /dev/null +++ b/src/crypto/sha3.cpp @@ -0,0 +1,128 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +// Based on https://github.com/mjosaarinen/tiny_sha3/blob/master/sha3.c +// by Markku-Juhani O. Saarinen + +#include +#include +#include + +#include +#include // For std::begin and std::end. + +#include + +// Internal implementation code. +namespace +{ +uint64_t Rotl(uint64_t x, int n) { return (x << n) | (x >> (64 - n)); } +} // namespace + +void KeccakF(uint64_t (&st)[25]) +{ + static constexpr uint64_t RNDC[24] = { + 0x0000000000000001, 0x0000000000008082, 0x800000000000808a, 0x8000000080008000, + 0x000000000000808b, 0x0000000080000001, 0x8000000080008081, 0x8000000000008009, + 0x000000000000008a, 0x0000000000000088, 0x0000000080008009, 0x000000008000000a, + 0x000000008000808b, 0x800000000000008b, 0x8000000000008089, 0x8000000000008003, + 0x8000000000008002, 0x8000000000000080, 0x000000000000800a, 0x800000008000000a, + 0x8000000080008081, 0x8000000000008080, 0x0000000080000001, 0x8000000080008008 + }; + static constexpr int ROTC[24] = { + 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 2, 14, + 27, 41, 56, 8, 25, 43, 62, 18, 39, 61, 20, 44 + }; + static constexpr int PILN[24] = { + 10, 7, 11, 17, 18, 3, 5, 16, 8, 21, 24, 4, + 15, 23, 19, 13, 12, 2, 20, 14, 22, 9, 6, 1 + }; + static constexpr int ROUNDS = 24; + + for (int round = 0; round < ROUNDS; ++round) { + uint64_t bc[5], t; + + // Theta + for (int i = 0; i < 5; i++) { + bc[i] = st[i] ^ st[i + 5] ^ st[i + 10] ^ st[i + 15] ^ st[i + 20]; + } + + for (int i = 0; i < 5; i++) { + t = bc[(i + 4) % 5] ^ Rotl(bc[(i + 1) % 5], 1); + for (int j = 0; j < 25; j += 5) st[j + i] ^= t; + } + + // Rho Pi + t = st[1]; + for (int i = 0; i < 24; i++) { + int j = PILN[i]; + bc[0] = st[j]; + st[j] = Rotl(t, ROTC[i]); + t = bc[0]; + } + + // Chi + for (int j = 0; j < 25; j += 5) { + for (int i = 0; i < 5; i++) bc[i] = st[j + i]; + for (int i = 0; i < 5; i++) { + st[j + i] ^= (~bc[(i + 1) % 5]) & bc[(i + 2) % 5]; + } + } + + // Iota + st[0] ^= RNDC[round]; + } +} + +SHA3_256& SHA3_256::Write(Span data) +{ + if (m_bufsize && m_bufsize + data.size() >= sizeof(m_buffer)) { + // Fill the buffer and process it. + std::copy(data.begin(), data.begin() + sizeof(m_buffer) - m_bufsize, m_buffer + m_bufsize); + data = data.subspan(sizeof(m_buffer) - m_bufsize); + m_state[m_pos++] ^= ReadLE64(m_buffer); + m_bufsize = 0; + if (m_pos == RATE_BUFFERS) { + KeccakF(m_state); + m_pos = 0; + } + } + while (data.size() >= sizeof(m_buffer)) { + // Process chunks directly from the buffer. + m_state[m_pos++] ^= ReadLE64(data.data()); + data = data.subspan(8); + if (m_pos == RATE_BUFFERS) { + KeccakF(m_state); + m_pos = 0; + } + } + if (data.size()) { + // Keep the remainder in the buffer. + std::copy(data.begin(), data.end(), m_buffer + m_bufsize); + m_bufsize += data.size(); + } + return *this; +} + +SHA3_256& SHA3_256::Finalize(Span output) +{ + assert(output.size() == OUTPUT_SIZE); + std::fill(m_buffer + m_bufsize, m_buffer + sizeof(m_buffer), 0); + m_buffer[m_bufsize] ^= 0x06; + m_state[m_pos] ^= ReadLE64(m_buffer); + m_state[RATE_BUFFERS - 1] ^= 0x8000000000000000; + KeccakF(m_state); + for (unsigned i = 0; i < 4; ++i) { + WriteLE64(output.data() + 8 * i, m_state[i]); + } + return *this; +} + +SHA3_256& SHA3_256::Reset() +{ + m_bufsize = 0; + m_pos = 0; + std::fill(std::begin(m_state), std::end(m_state), 0); + return *this; +} diff --git a/src/crypto/sha3.h b/src/crypto/sha3.h new file mode 100644 index 0000000000000..88d8c1204d957 --- /dev/null +++ b/src/crypto/sha3.h @@ -0,0 +1,41 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CRYPTO_SHA3_H +#define BITCOIN_CRYPTO_SHA3_H + +#include + +#include +#include + +//! The Keccak-f[1600] transform. +void KeccakF(uint64_t (&st)[25]); + +class SHA3_256 +{ +private: + uint64_t m_state[25] = {0}; + unsigned char m_buffer[8]; + unsigned m_bufsize = 0; + unsigned m_pos = 0; + + //! Sponge rate in bits. + static constexpr unsigned RATE_BITS = 1088; + + //! Sponge rate expressed as a multiple of the buffer size. + static constexpr unsigned RATE_BUFFERS = RATE_BITS / (8 * sizeof(m_buffer)); + + static_assert(RATE_BITS % (8 * sizeof(m_buffer)) == 0, "Rate must be a multiple of 8 bytes"); + +public: + static constexpr size_t OUTPUT_SIZE = 32; + + SHA3_256() {} + SHA3_256& Write(Span data); + SHA3_256& Finalize(Span output); + SHA3_256& Reset(); +}; + +#endif // BITCOIN_CRYPTO_SHA3_H diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 16f77264282cc..ec12aaadf7fc8 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -9,6 +9,7 @@ #include "crypto/ripemd160.h" #include "crypto/sha1.h" #include "crypto/sha256.h" +#include "crypto/sha3.h" #include "crypto/sha512.h" #include "crypto/hmac_sha256.h" #include "crypto/hmac_sha512.h" @@ -609,4 +610,110 @@ BOOST_AUTO_TEST_CASE(aes_cbc_testvectors) { "b2eb05e2c39be9fcda6c19078c6a9d1b3f461796d6b0d6b2e0c2a72b4d80e644"); } +static void TestSHA3_256(const std::string& input, const std::string& output) +{ + const auto in_bytes = ParseHex(input); + const auto out_bytes = ParseHex(output); + + SHA3_256 sha; + // Hash the whole thing. + unsigned char out[SHA3_256::OUTPUT_SIZE]; + sha.Write(in_bytes).Finalize(out); + assert(out_bytes.size() == sizeof(out)); + BOOST_CHECK(std::equal(std::begin(out_bytes), std::end(out_bytes), out)); + + // Reset and split randomly in 3 + sha.Reset(); + int s1 = InsecureRandRange(in_bytes.size() + 1); + int s2 = InsecureRandRange(in_bytes.size() + 1 - s1); + int s3 = in_bytes.size() - s1 - s2; + sha.Write(MakeSpan(in_bytes).first(s1)).Write(MakeSpan(in_bytes).subspan(s1, s2)); + sha.Write(MakeSpan(in_bytes).last(s3)).Finalize(out); + BOOST_CHECK(std::equal(std::begin(out_bytes), std::end(out_bytes), out)); +} + +BOOST_AUTO_TEST_CASE(keccak_tests) +{ + // Start with the zero state. + uint64_t state[25] = {0}; + CSHA256 tester; + for (int i = 0; i < 262144; ++i) { + KeccakF(state); + for (int j = 0; j < 25; ++j) { + unsigned char buf[8]; + WriteLE64(buf, state[j]); + tester.Write(buf, 8); + } + } + uint256 out; + tester.Finalize(out.begin()); + // Expected hash of the concatenated serialized states after 1...262144 iterations of KeccakF. + // Verified against an independent implementation. + BOOST_CHECK_EQUAL(out.ToString(), "5f4a7f2eca7d57740ef9f1a077b4fc67328092ec62620447fe27ad8ed5f7e34f"); +} + +BOOST_AUTO_TEST_CASE(sha3_256_tests) +{ + // Test vectors from https://csrc.nist.gov/CSRC/media/Projects/Cryptographic-Algorithm-Validation-Program/documents/sha3/sha-3bytetestvectors.zip + + // SHA3-256 Short test vectors (SHA3_256ShortMsg.rsp) + TestSHA3_256("", "a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a"); + TestSHA3_256("e9", "f0d04dd1e6cfc29a4460d521796852f25d9ef8d28b44ee91ff5b759d72c1e6d6"); + TestSHA3_256("d477", "94279e8f5ccdf6e17f292b59698ab4e614dfe696a46c46da78305fc6a3146ab7"); + TestSHA3_256("b053fa", "9d0ff086cd0ec06a682c51c094dc73abdc492004292344bd41b82a60498ccfdb"); + TestSHA3_256("e7372105", "3a42b68ab079f28c4ca3c752296f279006c4fe78b1eb79d989777f051e4046ae"); + TestSHA3_256("0296f2c40a", "53a018937221081d09ed0497377e32a1fa724025dfdc1871fa503d545df4b40d"); + TestSHA3_256("e6fd42037f80", "2294f8d3834f24aa9037c431f8c233a66a57b23fa3de10530bbb6911f6e1850f"); + TestSHA3_256("37b442385e0538", "cfa55031e716bbd7a83f2157513099e229a88891bb899d9ccd317191819998f8"); + TestSHA3_256("8bca931c8a132d2f", "dbb8be5dec1d715bd117b24566dc3f24f2cc0c799795d0638d9537481ef1e03e"); + TestSHA3_256("fb8dfa3a132f9813ac", "fd09b3501888445ffc8c3bb95d106440ceee469415fce1474743273094306e2e"); + TestSHA3_256("71fbacdbf8541779c24a", "cc4e5a216b01f987f24ab9cad5eb196e89d32ed4aac85acb727e18e40ceef00e"); + TestSHA3_256("7e8f1fd1882e4a7c49e674", "79bef78c78aa71e11a3375394c2562037cd0f82a033b48a6cc932cc43358fd9e"); + TestSHA3_256("5c56a6b18c39e66e1b7a993a", "b697556cb30d6df448ee38b973cb6942559de4c2567b1556240188c55ec0841c"); + TestSHA3_256("9c76ca5b6f8d1212d8e6896ad8", "69dfc3a25865f3535f18b4a7bd9c0c69d78455f1fc1f4bf4e29fc82bf32818ec"); + TestSHA3_256("687ff7485b7eb51fe208f6ff9a1b", "fe7e68ae3e1a91944e4d1d2146d9360e5333c099a256f3711edc372bc6eeb226"); + TestSHA3_256("4149f41be1d265e668c536b85dde41", "229a7702448c640f55dafed08a52aa0b1139657ba9fc4c5eb8587e174ecd9b92"); + TestSHA3_256("d83c721ee51b060c5a41438a8221e040", "b87d9e4722edd3918729ded9a6d03af8256998ee088a1ae662ef4bcaff142a96"); + TestSHA3_256("266e8cbd3e73d80df2a49cfdaf0dc39cd1", "6c2de3c95900a1bcec6bd4ca780056af4acf3aa36ee640474b6e870187f59361"); + TestSHA3_256("a1d7ce5104eb25d6131bb8f66e1fb13f3523", "ee9062f39720b821b88be5e64621d7e0ca026a9fe7248d78150b14bdbaa40bed"); + TestSHA3_256("d751ccd2cd65f27db539176920a70057a08a6b", "7aaca80dbeb8dc3677d18b84795985463650d72f2543e0ec709c9e70b8cd7b79"); + TestSHA3_256("b32dec58865ab74614ea982efb93c08d9acb1bb0", "6a12e535dbfddab6d374058d92338e760b1a211451a6c09be9b61ee22f3bb467"); + TestSHA3_256("4e0cc4f5c6dcf0e2efca1f9f129372e2dcbca57ea6", "d2b7717864e9438dd02a4f8bb0203b77e2d3cd8f8ffcf9dc684e63de5ef39f0d"); + TestSHA3_256("d16d978dfbaecf2c8a04090f6eebdb421a5a711137a6", "7f497913318defdc60c924b3704b65ada7ca3ba203f23fb918c6fb03d4b0c0da"); + TestSHA3_256("47249c7cb85d8f0242ab240efd164b9c8b0bd3104bba3b", "435e276f06ae73aa5d5d6018f58e0f009be351eada47b677c2f7c06455f384e7"); + TestSHA3_256("cf549a383c0ac31eae870c40867eeb94fa1b6f3cac4473f2", "cdfd1afa793e48fd0ee5b34dfc53fbcee43e9d2ac21515e4746475453ab3831f"); + TestSHA3_256("9b3fdf8d448680840d6284f2997d3af55ffd85f6f4b33d7f8d", "25005d10e84ff97c74a589013be42fb37f68db64bdfc7626efc0dd628077493a"); + TestSHA3_256("6b22fe94be2d0b2528d9847e127eb6c7d6967e7ec8b9660e77cc", "157a52b0477639b3bc179667b35c1cdfbb3eef845e4486f0f84a526e940b518c"); + TestSHA3_256("d8decafdad377904a2789551135e782e302aed8450a42cfb89600c", "3ddecf5bba51643cd77ebde2141c8545f862067b209990d4cb65bfa65f4fa0c0"); + TestSHA3_256("938fe6afdbf14d1229e03576e532f078898769e20620ae2164f5abfa", "9511abd13c756772b852114578ef9b96f9dc7d0f2b8dcde6ea7d1bd14c518890"); + TestSHA3_256("66eb5e7396f5b451a02f39699da4dbc50538fb10678ec39a5e28baa3c0", "540acf81810a199996a612e885781308802fe460e9c638cc022e17076be8597a"); + TestSHA3_256("de98968c8bd9408bd562ac6efbca2b10f5769aacaa01365763e1b2ce8048", "6b2f2547781449d4fa158180a178ef68d7056121bf8a2f2f49891afc24978521"); + TestSHA3_256("94464e8fafd82f630e6aab9aa339d981db0a372dc5c1efb177305995ae2dc0", "ea7952ad759653cd47a18004ac2dbb9cf4a1e7bba8a530cf070570c711a634ea"); + TestSHA3_256("c178ce0f720a6d73c6cf1caa905ee724d5ba941c2e2628136e3aad7d853733ba", "64537b87892835ff0963ef9ad5145ab4cfce5d303a0cb0415b3b03f9d16e7d6b"); + TestSHA3_256("14365d3301150d7c5ba6bb8c1fc26e9dab218fc5d01c9ed528b72482aadee9c27bef667907797d55514468f68791f053daa2df598d7db7d54beea493bdcbb0c75c7b36ad84b9996dca96354190bd96d9d7fbe8ff54ffaf77c55eb92985da50825ee3b4179f5ec88b6fa60bb361d0caf9493494fe4d28ef843f0f498a2a9331b82a", "9b690531dee948a9c559a2e0efab2ec824151a9175f2730a030b748d07cbaa7f"); + TestSHA3_256("4a757db93f6d4c6529211d70d5f8491799c0f73ae7f24bbd2138db2eaf2c63a85063b9f7adaa03fc348f275323248334e3ffdf9798859f9cf6693d29566ff7d50976c505ecb58e543c459b39acdf4ce4b5e80a682eaa7c1f1ce5fe4acb864ff91eb6892b23165735ea49626898b40ceeb78161f5d0ea4a103cb404d937f9d1dc362b", "1ac7cc7e2e8ea14fb1b90096f41265100712c5dd41519d78b2786cfb6355af72"); + TestSHA3_256("da11c39c77250f6264dda4b096341ff9c4cc2c900633b20ea1664bf32193f790a923112488f882450cf334819bbaca46ffb88eff0265aa803bc79ca42739e4347c6bff0bb9aa99780261ffe42be0d3b5135d03723338fb2776841a0b4bc26360f9ef769b34c2bec5ed2feb216e2fa30fa5c37430c0360ecbfba3af6fb6b8dedacbb95c", "c163cd43de224ac5c262ae39db746cfcad66074ebaec4a6da23d86b310520f21"); + TestSHA3_256("3341ca020d4835838b0d6c8f93aaaebb7af60730d208c85283f6369f1ee27fd96d38f2674f316ef9c29c1b6b42dd59ec5236f65f5845a401adceaa4cf5bbd91cac61c21102052634e99faedd6cdddcd4426b42b6a372f29a5a5f35f51ce580bb1845a3c7cfcd447d269e8caeb9b320bb731f53fe5c969a65b12f40603a685afed86bfe53", "6c3e93f2b49f493344cc3eb1e9454f79363032beee2f7ea65b3d994b5cae438f"); + TestSHA3_256("989fc49594afc73405bacee4dbbe7135804f800368de39e2ea3bbec04e59c6c52752927ee3aa233ba0d8aab5410240f4c109d770c8c570777c928fce9a0bec9bc5156c821e204f0f14a9ab547e0319d3e758ae9e28eb2dbc3d9f7acf51bd52f41bf23aeb6d97b5780a35ba08b94965989744edd3b1d6d67ad26c68099af85f98d0f0e4fff9", "b10adeb6a9395a48788931d45a7b4e4f69300a76d8b716c40c614c3113a0f051"); + TestSHA3_256("e5022f4c7dfe2dbd207105e2f27aaedd5a765c27c0bc60de958b49609440501848ccf398cf66dfe8dd7d131e04f1432f32827a057b8904d218e68ba3b0398038d755bd13d5f168cfa8a11ab34c0540873940c2a62eace3552dcd6953c683fdb29983d4e417078f1988c560c9521e6f8c78997c32618fc510db282a985f868f2d973f82351d11", "3293a4b9aeb8a65e1014d3847500ffc8241594e9c4564cbd7ce978bfa50767fe"); + TestSHA3_256("b1f6076509938432145bb15dbe1a7b2e007934be5f753908b50fd24333455970a7429f2ffbd28bd6fe1804c4688311f318fe3fcd9f6744410243e115bcb00d7e039a4fee4c326c2d119c42abd2e8f4155a44472643704cc0bc72403b8a8ab0fd4d68e04a059d6e5ed45033b906326abb4eb4147052779bad6a03b55ca5bd8b140e131bed2dfada", "f82d9602b231d332d902cb6436b15aef89acc591cb8626233ced20c0a6e80d7a"); + TestSHA3_256("56ea14d7fcb0db748ff649aaa5d0afdc2357528a9aad6076d73b2805b53d89e73681abfad26bee6c0f3d20215295f354f538ae80990d2281be6de0f6919aa9eb048c26b524f4d91ca87b54c0c54aa9b54ad02171e8bf31e8d158a9f586e92ffce994ecce9a5185cc80364d50a6f7b94849a914242fcb73f33a86ecc83c3403630d20650ddb8cd9c4", "4beae3515ba35ec8cbd1d94567e22b0d7809c466abfbafe9610349597ba15b45"); + + // SHA3-256 Long test vectors (SHA3_256LongMsg.rsp) + TestSHA3_256("b1caa396771a09a1db9bc20543e988e359d47c2a616417bbca1b62cb02796a888fc6eeff5c0b5c3d5062fcb4256f6ae1782f492c1cf03610b4a1fb7b814c057878e1190b9835425c7a4a0e182ad1f91535ed2a35033a5d8c670e21c575ff43c194a58a82d4a1a44881dd61f9f8161fc6b998860cbe4975780be93b6f87980bad0a99aa2cb7556b478ca35d1f3746c33e2bb7c47af426641cc7bbb3425e2144820345e1d0ea5b7da2c3236a52906acdc3b4d34e474dd714c0c40bf006a3a1d889a632983814bbc4a14fe5f159aa89249e7c738b3b73666bac2a615a83fd21ae0a1ce7352ade7b278b587158fd2fabb217aa1fe31d0bda53272045598015a8ae4d8cec226fefa58daa05500906c4d85e7567", "cb5648a1d61c6c5bdacd96f81c9591debc3950dcf658145b8d996570ba881a05"); + TestSHA3_256("712b03d9ebe78d3a032a612939c518a6166ca9a161183a7596aa35b294d19d1f962da3ff64b57494cb5656e24adcf3b50e16f4e52135d2d9de76e94aa801cf49db10e384035329c54c9455bb3a9725fd9a44f44cb9078d18d3783d46ce372c31281aecef2f8b53d5702b863d71bc5786a33dd15d9256103b5ff7572f703d5cde6695e6c84f239acd1d6512ef581330590f4ab2a114ea064a693d5f8df5d908587bc7f998cde4a8b43d8821595566597dc8b3bf9ea78b154bd8907ee6c5d4d8a851f94be510962292b7ddda04d17b79fab4c022deb400e5489639dbc448f573d5cf72073a8001b36f73ac6677351b39d9bdb900e9a1121f488a7fa0aee60682e7dc7c531c85ec0154593ded3ae70e4121cae58445d8896b549cacf22d07cdace7625d57158721b44851d796d6511c38dac28dd37cbf2d7073b407fbc813149adc485e3dacee66755443c389d2d90dc70d8ff91816c0c5d7adbad7e30772a1f3ce76c72a6a2284ec7f174aefb6e9a895c118717999421b470a9665d2728c3c60c6d3e048d58b43c0d1b5b2f00be8b64bfe453d1e8fadf5699331f9", "095dcd0bc55206d2e1e715fb7173fc16a81979f278495dfc69a6d8f3174eba5a"); + TestSHA3_256("2a459282195123ebc6cf5782ab611a11b9487706f7795e236df3a476404f4b8c1e9904e2dc5ef29c5e06b179b8649707928c3913d1e53164747f1fa9bba6eeaf8fb759d71e32adc8c611d061345882f1cdeee3ab4cab3554adb2e43f4b01c37b4546994b25f4dcd6c497bc206865643930157cb5b2f4f25be235fa223688535907efcc253bcd083021407ea09cb1c34684aa0c1849e7efe2d9af6938c46525af9e5afb4da6e5b83da4b61dc718672a8090549cbe5aadb44f5bc93a6b3fbdc2e6d32e2eaaae637465179ea17f23ad1e4f1ebc328e2c6dc90c302b74a1edbbb0676c136b269d70c41040a313af06ab291bf489d9700950b77f207c1fc41884799931b3bca8b93331a6e96b7a3f0a8bd24cdb64964c377e0512f36444bb0643a4e3ecb328194cd5428fd89ede167472a14a9bf5730aff1e3b2c708de96eff1ebaaf63beb75f9c7d8034d6e5471e8f8a1f7efce37793a958e134619c19c54d3d42645f7a7263f25471fbaae8be3ea2fbd34ec6d7aacd7d5680948c3cd9a837c9c469a88f600d95829f4d1e4e4a5ef4ed4623c07815a1c33d9fb3b91333ff04eac92806a68a46cf2e9293f8bff466ce87fe66b46fbff7c238c7f9b2c92eb2fdc7d8084167f6f4e680d03301e5c33f78f1857d6863b1b8c36c7fce3e07d2a96a8979712079ae0023a1e3970165bfcf3a5463d2a4fdf1ca0e044f9a247528cd935734cb6d85ba53ceb95325c0eaf0ff5cd81ecb32e58917eb26bfc52dba3704bf5a927fee3220", "cb1c691c87244c0caf733aacd427f83412cd48820b358c1b15dd9fadee54e5af"); + TestSHA3_256("32659902674c94473a283be00835eb86339d394a189a87da41dad500db27da6b6a4753b2bb219c961a227d88c6df466ba2fc1e9a2d4c982db4398778c76714d5e9940da48bc3808f3c9989131a07683b8c29d6af336e9aee1dfa57d83c48a86f17146edec07869bb06550689ebf4788159ed0a921048b4a6e3e3ec272413bec15d8e1f6a40897fa0e11d9df223ef9fc270106249ae220fdc6ebdef6d6611805421ccc850f53ee9c836baf657a94005883b5a85def344d218264f07b2ea8714afcc941096c6ded0bb6bf5b8bf652fd15a21931c58c9f526e27363ddff98c0a25bc7af9f469ab35bffea948b333f042cc18a82cec0177f33c3bdbf185b580353de79e51e675b03b31e195f19ba1f063d44def0441dc52820426c2c61cf12974ec249fd3502f017ffa06220075ced7e2d6b86a52677ba3916e8e8726062aec5bc8ea1c18b1e4137680b2c9d002191b423bee8691bd7e0f93c3b9959bc1c14d5c5cbe8f7c9c336aa16e9de9faa12f3f048c66d04cb441eb2bbc5e8a91e052c0f9000856896f9b7ba30c1e2eead36fc7ac30a7d3ddfc65caaba0e3b292d26dfba46b5e2dc9bc9acadde1c9f52b2969299bd1281ddff65822b629cfba2928613200e73661b803afdcc4a817d9361389e975e67dfadd22a797bdaf991ddf42db18711c079ecec55925f9978e478612609bacd900172011c27e24bad639ffc24a23877278318872153aef6893ccb5b68b94b33154df7334375aadd3edbb35272cc7b672dec68faa62900873ded52f6049891b77f2d0311a84b19b73660e09d1f1998095c1da1edecfa9f741b5fd6db048dd68255085d43529279021d59ed853470d6863b7c8e07fcb0d1e6acfb1eb16f7f60bb1f46ce70493010e57930a3b4b8b87e065272f6f1dd31df057627f4214e58798b664e1e40960f2789d44ccacfb3dbd8b02a68a053976711f8034c1ed3a8", "5ac9275e02543410359a3f364b2ae3b85763321fd6d374d13fe54314e5561b01"); + TestSHA3_256("a65da8277a3b3738432bca9822d43b3d810cdad3b0ed2468d02bd269f1a416cd77392190c2dde8630eeb28a297bda786017abe9cf82f14751422ac9fff6322d5d9a33173db49792d3bc37fff501af667f7ca3dd335d028551e04039ef5a9d42a9443e1b80ea872fd945ad8999514ae4a29a35f60b0f7e971b67ae04d1ba1b53470c03847a3225c3ddf593a57aed3599661ae2d2bb1cddd2fa62c4a94b8704c5c35c33e08e2debe54e567ae21e27e7eb36593ae1c807a8ef8b5c1495b15412108aaf3fce4130520aa6e2d3bdf7b3ea609fdf9ea1c64258435aae2e58a7b3abda198f979c17dbe0aa74253e979bf3a5800f388ea11a7f7454c4e36270a3083a790c77cbe89693205b32880c0d8f79b1c000ee9b5e58f175ba7696616c17c45673cff25d1221f899836e95cc9e26a887a7115c4537e65ad4eacc319ba98a9a8860c089cbc76e7ea4c984d900b80622afbbbd1c0cdc670e3a4c523f81c77fed38b6aa988876b097da8411cc48e9b25a826460a862aa3fadfe75952aa4347c2effebdac9138ebcc6c34991e9f5b19fc2b847a87be72ff49c99ecf19d837ee3e23686cd760d9dd7adc78091bca79e42fdb9bc0120faec1a6ca52913e2a0156ba9850e1f39d712859f7fdf7daedf0e206dff67e7121e5d1590a8a068947a8657d753e83c7f009b6b2e54acc24afc9fdc9601a1d6d9d1f17aab0ce96c4d83405d1e3baba1dffa86ecccee7f1c1b80b1bbf859106ce2b647ae1e4a6a9b584ae1dfc0a4deebb755638f1d95dcc79b1be263177e2a05c72bde545d09ba726f41d9547117e876af81bfc672e33c71442eb05675d9552df1b313d1f9934f9ddd08955fa21d6edf23000a277f6f149591299a0a96032861ecdc96bb76afa05a2bffb445d61dc891bc70c13695920b911cad0df3fa842a3e2318c57556974343f69794cb8fa18c1ad624835857e4781041198aa705c4d11f3ef82e941be2aee7a770e54521312fe6facbaf1138eee08fa90fae986a5d93719aeb30ac292a49c1d91bf4574d553a92a4a6c305ab09db6bbeffd84c7aa707f1c1628a0220d6ba4ee5e960566686228a6e766d8a30dddf30ed5aa637c949950c3d0e894a7560670b6879a7d70f3c7e5ab29aed236cc3527bdea076fec8add12d784fbcf9a", "68f62c418a6b97026cc70f6abf8419b671ee373709fa13074e37bd39f0a50fcb"); + TestSHA3_256("460f8c7aac921fa9a55800b1d04cf981717c78217cd43f98f02c5c0e66865c2eea90bcce0971a0d22bc1c74d24d9bfea054e558b38b8502fccb85f190d394f2f58f581a02d3b9cc986f07f5a67d57ab4b707bd964ecc10f94f8cc538b81eeb743746c537407b7b575ced0e1ec4c691a72eb0978be798e8be22b278b390be99c730896fdc69b6a44456be5ee261366e8b1351cbb22aa53e45ec325ed2bca0bfeeebc867d7d07681581b6d56ed66ac78280df04053407a7b57561261dd644cc7b20f0a95709e42795b5402dd89fcb11746c597e0b650a008bc085c681bb24b17db4458e1effba3f414a883ddfc4bccb3ace24d9223839d4b3ca9185ad5cc24193134b9339b0e205a4cc0fa3d8f7a85b4230d1b3ee101fbae9ee14c2153da5f337c853573bd004114cb436ee58ab1648373ee07cc39f14198ac5a02a4dd0585cf83dfd4899df88e8859dae8bc351af286642c1c25737bf8712cb941cbbb741d540feb9f5d831f901fbe2d6facd7dab626bd705f2fd7c9a7a0e7a9127e3451af2ae8509dd7b79dce41c1e30b9dba1c38cb4861dad3ac00d68fa5d07ba591c1c3b9d6b7d6e08099d0572ca4c475240601decba894fa3c4b0ea52ed687281beee268a1c8535e283b1fc7c51aa31d5ec098c50fec958acdd0d54a49643bef170093a1102a1b3bf5ad42fb55ebaf7db07385eadcd6e66da8b7b6e6c022a1e3d01f5fccec86365d3014c159a3bff17d614751b3fa0e8e89152936e159b7c0ea8d71cd4ffd83adae209b254b793f6f06bb63838a303b95c85b4edfa4ddcca0ed952165930bca87140f67f5389d1233fe04f0a3d647050410c44d389513084ad53155af00de02cc7943a3b988d8e1454f85153aff0816e24b964ec91dc514c588a93634ff3dd485c40575faa2f254abdf86fbcf6d381337601a7b1ba5b99719f045eb7bf6f2e8b9dd9d053ef0b3126f984fc9ea87a2a70b3798fab593b83a4ff44d9c0c4ec3e570ac537c10d9e3c4996027a813b70d7867b858f31f508aa56e7b087370707974b2186f02f5c549112f2158c0d365402e52cba18fe245f77f7e6fbf952ec2dc3c880b38be771caea23bc22838b1f70472d558bdf585d9c77088b7ba2dceaeb3e6f96df7d91d47da1ec42be03936d621ecf747f24f9073c122923b4161d99bc8190e24f57b6fac952ed344c7eae86a5f43c08089c28c7daf3aa7e39c59d6f1e17ece1977caf6b4a77a6ff52774521b861f38ebc978005e5763cc97123e4d17c7bc4134c8f139c7d7a9a02646fef9525d2a6871fc99747e81430b3fec38c677427c6f5e2f16c14eee646ebf6eb16775ad0957f8684c7045f7826bc3736eca", "7d495ddf961cbff060f80b509f2b9e20bed95319eef61c7adb5edeec18e64713"); + TestSHA3_256("c8a2a26587d0126abe9ba8031f37d8a7d18219c41fe639bc7281f32d7c83c376b7d8f9770e080d98d95b320c0f402d57b7ef680da04e42dd5211aacf4426ecca5050ca596312cfae79cee0e8c92e14913cc3c66b24ece86c2bfa99078991faad7b513e94f0b601b7853ddb1eb3c9345f47445a651389d070e482ea5db48d962820257daf1cbe4bb8e5f04a3637d836c8c1bc4d83d6eda5f165f2c2592be268412712ae324ef054bb812f56b8bc25c1d59071c64dd3e00df896924c84575817027861faa5f016c5c74142272daa767e8c9dacee4c732ab08b5fa9ad65a0b74c73fb5a889169f645e50d70e41d689415f7d0b4ec071e9238b5a88110856fc6ae9b9944817e21597d1ccd03b60e60472d1e11d3e9063de24a7b59609b6a2a4ee68238690cf2800614746941c48af9566e07494f0dd236e091e75a8f769e3b179b30c10f5277eec7b3f5c97337189b8b82bc5e717ff27355b2009356caa908e976ae1d7f7a94d36202a8d5e03641aeac0e453a8168ee5a0858ceecfcbf11fb8c1f033201add297a0a89476d2ea8b9a82bda8c3c7ef4f55c3295a4ecb7c607ac73d37eadc13b7a2494ec1928f7a80c8d534efe38a3d9ccb4ccdab9f092a1def6478532c5ad3cd5c259b3812600fa89e6d1e228114795d246cedc9c9fff0d1c1297a5ddfc1169c2efb3800df8dd18a8511214785abcc1bc7eb31bdb2f5f70358dfe860ed5a03ab7e95cc21df5ee7aee68be568d6985e5c1e91408e4432663b1c4e6d613d6dc382b5b900a4fc1b7a9c27a1138c5e2356ab9026c34465006602753daf6ab7427da93c307c901d0bb1ddb21c53bc0493dd8d857161e8ffa51fdecb75568243205aa979c2e7ed2a77b5f8edc34cffb0321a8c653bc381f96ab85a86bf0bb2c9518208d636eac40aa7ad754260a75d4a46362f994c90173b975afb0ee17601311b1c51ba562c1ca7e3c2dd18b90bdebb1858fe876c71b3ad742c4bcba33e7763c750098de856fde8731cb6d698218be9f0a98298630e5b374957d126cf0b1c489c48bab6b50f6fb59ee28be6c3916bbd16514234f80e1ac15d0215852b87f9c6e429eb9f85007bf6ae3de1af0202861fd177c7c4f51af533f956a051815815c6e51e25af20d02893e95442991f1de5f86a4397ae20d9f675657bf9f397267831e94cef4e4d287f759850350ce0898f2e29de3c5c41f4246fe998a8d1359a2bed36ded1e4d6b08682025843700fee8cab56703e342212870acdd53655255b35e414fa53d9810f47a37195f22d72f6e555392023a08adc282c585b2ae62e129efccdc9fe9617eecac12b2ecdabd247a1161a17750740f90ebed3520ceb17676f1fa87259815ff415c2794c5953f689c8d5407dbbd10d1241a986e265cea901af34ec1ded0323ca3290a317208ba865637af4797e65b9cfcad3b931bbf6ac896623e2f4408529172911f1b6a9bcae8279ec7e33452d0cd7b026b46a99cbe8a69cd4d21cdc6d3a84002fab527c4fd18a121526d49890ced3fb89beb384b524015a2e03c049241eb9", "b8d4b29b086ef6d6f73802b9e7a4f2001e384c8258e7046e6779662fd958517e"); + TestSHA3_256("3a86a182b54704a3af811e3e660abcfbaef2fb8f39bab09115c1068976ff694bb6f5a3839ae44590d73e4996d45af5ceb26b03218ab3fef6f5f4ef48d22839fb4371c270f9535357b22142c4ffb54e854b64cab41932fe888d41ca702e908c63eae244715bfbf69f481250f16f848dc881c6996e6f9d76f0e491de2c129f2a2ab22e72b04644f610a2fabc45aa2d7b3e5d77b87a135d2fd502ca74a207bddaf9a43e945245961a53c7bfcfe73a1ae090e6606ffe8ddbf1e0f0d6d4fa94526578c6faf282dd592b10bf4bce00a7b1846625690623667e83b9b59b465d42c6944e224ad36698f5f2ee938404b7775c2e66207bc41025adaf07590312f398812d24c0178126fdd334964a54b8353482a83be17cf2ee52d23b72e5f57fe31eebf8a1a64742eb9459bcb0eca231a1658ab88b7056d8e47554f0a46058d6565c6cbf6edec45fdde6f051e38255b82493de27ffd3efbe1b179b9642d2166073db6d4832707420237a00bad7125795e645e5bc3e1431ecbabf0ff5f74416626322545c966241cce6d8f2c035a78f100e030741f13b02a9eaf618d468bc40274db98bc342be12ad4d892c2ba546e571c556ac7cbf4e4c3fd3431efd40457cf65a297845dd8cce09811418c3cef941ff32c43c375157f6f49c2e893625e4b216b1f985aa0fd25f29a9011d4f59c78b037ed71f384e5de8116e3fc148c0a3cad07cb119b9829aac55eed9a299edb9abc5d017be485f690add70ff2efbb889ac6ce0da9b3bdbeb9dd47823116733d58a8d510b7f2e2c8244a2cbf53816b59e413207fb75f9c5ce1af06e67d182d3250ea3283bcbb45cb07ea6a6aa486361eb6f69199c0eb8e6490beff82e4ab274b1204e7f2f0ba097fba0332aa4c4a861771f5b3d45ce43e667581a40fee4bebe7fa9d87b70a5bb876c928f7e6d16ae604b3a4e9c7f1d616e2deab96b6207705b9a8f87468503cdd20a3c02cc8da43d046da68b5ed163d926a5a714a4df1b8ef007bca408f68b9e20de86d6398ad81df5e74d5aaac40874b5d6787211ff88e128cf1676e84ca7f51aee5951efee1915dcc11502a8df74fac4c8451dda49b631a8fb87470f0ebe9b67449bbd1640ceee6101e8cd82aa1033fa84f75b28450e461b93f65da5c43759b0e83660d50961702bb1ad015dad42e600117475237cf6e7279d4a02d1f67cf59de0108355d03963e3d84ce7647173dd7d77a6b3f275d7de74236d7bbb2df437d536136dbe1dbe8f307facc7bc7d0cde1abf745cbeb81af1ab2c46138cf007e901f22668377958bcbbadb7e9905973b27ff0c5baaece25e974c1bd116cc81dd1c81a30bae86a6fb12c6a5494068e122153128313eb3e628d76e9babc823c9eb9d3b81bacfa7a6b372abe6b1246a350f23e2e95b09c9037a75aac255ef7d4f267cad3ce869531b4165db2e5a9792094efea4ae3d9ea4d0efdc712e63df21882a353743190e016b2166e4da8a2c78e48defc7155d5fdfc4e596624e6a19c91b43719a22c1204b1cefe05989d455773d3881fa8d3eefc255f81dfe90bd41dc6f1e9c265a753298a6e98c999acd9525a9db5f9f9456a0f51a93dd9693e1d9c3fa283f7c58a9c752afcaa635abea8dfc80e2c326b939260069457fdad68c341852dcb5fcbbd351318defd7ae3b9f827478eb77306a5ae14cf8895f2bc6f0f361ffc8aa37e286629dc7e59b73a8712525e851c64d363065631edc1609f3d49a09575876a", "b71ec00c0fcc4f8663312711540df1cd236eb52f237409415b749ff9436dc331"); + TestSHA3_256("c041e23b6d55998681802114abc73d2776967cab715572698d3d497ec66a790b0531d32f45b3c432f5b2d8039ea47de5c6060a6514f3ff8fb5f58e61fd1b5b80524c812a46dad56c035a6e95ecb465ea8176d99b836e36f65977b7dbb3932a706d3af415b6f2549b7120ecb0db1e7d9e6f8df23607eda006436bccd32ef96d431fa434d9de22ca2608ab593eb50b4d6a57f45c1ce698c3283a77d330b876ad6030324a5c0693be7790a4bd26c0a25eb403531f37689829c20546d6dc97327131688b3d88766db8f5d1b22050450c37e53951446dd7155a3e6d7edcbe1354411d8f58154475d74008937e8ba48b706066c296d1a87936dd023ac8eebe7605a58c6c40da774cf9df189db0050adcf7629e66cbd1cf9824397834cb13c4066c26e6c8ec950b44fc1c8db8ef976a7ec8c4f4ec9849ca7a07f906223053b80db24b946b034ee7a30880d0ace348acba0d0ed21ea443816706a216ce9eb682d1fe9dfc1d2e0bf3b1449247413520b8d8ebc99fc298c6dca949be0ffebe450b9b79a387a615d617b8d9da5b3e8d2776208c7cc2a11bdbc387f9d4597b380739b24ae59dcd5fb63bfeefe0746d9266cfda18afa583d6891e483e6d5c0db305f5609beba75bb5b447ccac2dfb94ede4a94db6eaaf3070d8d5353f107f7bd74528eb913e0b19bed6236a3b48567c46a9eec28fb6486f92d0d09625452d8f4dd1b89c566533cc2326b820c2b9efed43be8481cb9ad809e47af7b31795cb0fbdb18fbb12e8853f8bacec366a092daf8f2a55d2911fc7c70ddd33d33e86c2c4ceeb9390ec506b399f6fa8f35abf7789d0f547fd09cb7e6fb6016a3fc2a27a762989ae620d234c810777d5a1bb633744af2844495d2963c986ef8540ca715bed7692c77b9dec90e06acc5986b47dd4a8d3ca3300b2bedf9f26ae6d1c7e7acef05c0fc521c3309e1e70771eea6e96b67de5e3fb6833145bb73d46081b074539498307929da779e003c27f0a171035458b8c7c86c905b23dda74c040878d5a05be94821537724ebd5608ec0754c3e3e99a719bbb6d5320eed07323fca637429b18378936364c389de1e9c6fce8af270a713b4b829b43e7d761e17724c22e84611e1322dde45cbee86a0e16d01cfb8910d99391c39afd8e5f5567c59f219aa8c19ad158f287cb6807ba1fe46d38d091639a217766b3aa9ded73ac14570ba236225218305d68c0be6099c336ba8455c86b7c8c04542e729ceb84596c33ca6eb7ec1091b406cf64495ccfa2169f47b3b590477d4073537c14c05015d51ba527b3869ae4ebd603df906323658b04cb11e13bc29b34ac69f18dd49f8958f7e3f5b05ab8b8ddb34e581bde5eb49dd15698d2d2b68fe7e8baf88d8f395cfcafcdff38cf34b59386f6f77333483655ee316f12bfeb00610d8cba9e59e637ca2cab6ed24dd584143844e61fcca994ba44a4c029682997ab04285f479a6dc2c854c569073c62cd68af804aa70f4976d5b9f6b09d3738fcccb6d60e11ba97a4001062195d05a43798d5f24e9466f082ac367169f892dfd6cc0adeef82212c867a49cba65e0e636bab91e2176d3865634aa45b13c1e3e7cdb4e7872b2437f40f3de5493792c06611a9ca97d0baed71bfb4e9fdd58191198a8b371aea7f65b6e851ce22f4808377d09b6a5a9f04eddc3ff4ef9fd8bf043bb559e1df5319113cb8beea9e06b0c05c50885873acd19f6e8a109c894403a415f627cd1e8f7ca54c288c230795aaddde3a787c2a20ac6dee4913da0240d6d971f3fce31fc53087afe0c45fa3c8f744c53673bec6231abe2623029053f4be0b1557e00b291ebb212d876e88bcc81e5bd9eb820691dca5fbdcc1e7a6c58945a2cac8db2d86c2a7d98dc5908598bda78ce202ac3cd174d48ad9cac9039e27f30658eef6317cd87c199944343e7fce1b3ea7", "ad635385a289163fbaf04b5850285bfe3759774aee7fd0211d770f63985e1b44"); + TestSHA3_256("01ec0bfc6cc56e4964808e2f1e516416717dad133061e30cb6b66b1dc213103b86b3b017fa7935457631c79e801941e3e3a0e1a3016d435e69a390eaac64f3166d944c8eb8df29fe95fdf27adc34631e4a1f3ff1d5af430f3d6f5908e40c0f83df1447274dfe30bbe76b758bd9abb40ed18331c7552dcc6959a1303e11134ec904bd0aab62de33c39703b99920851afd9d531eeb28f1c4b2e6c17c55db8296320316fbe19e881b5fcb4d266c58ca7f31d9176e26f70315330b58a516ec60d10404a78393aa03ced7acd225cb2a83caf3ab5888406a69a534f1ed1346e9b5e68831f90b872d57367361191c803eb7e38b3b9cd601282d5efdbf082db07d89bd06b093f986d08d3a7b12aa74513b6eb241b26ebf31da5726d59e315d1b4ee53ec6a9fdb6583bacc136e90e9607cab01e5d3853ab9727ede706b6f10b4e04d0510f45c0abc515bcb5ed0bcce86a92861126f4d502fcb8f988d62ecf9d124853de2bab633f9506c6fde8a36cd4413cf773e50f7b2d283482f18e2f547c2fc275cd60056ed98fb8d0816fd777c1566f0c2ae3b1cd92e344910a75e006106d193e06f7786ae37dd0e529cacf74176fd4cc1f6500549af5902dbbd56a70c194f5b671372edec425f90add40b4eb3d55123f3ab62797ad25bf5eecf4f417f86b00e6f76a4f52e44fd949851aae649dd0d26d641d4c1f343c7a2c851ca7851bbbdfd57ed6024eabc518a909a1e4689ea7bc5f83e19872950368a06e93ab41944c3d8befc5705b814e5f33511a7f7ea8a4771c804b321a3a3f32c18fa127d3c9e6c011337dc100ceb156ed45d0a62f238dacac44a3429f89bb7f98d09043c42451106e30471cc6fab7a4e1ce0a8202772b0218b631f287ec3ef82b1aa6299a0b54d6aad06aa9346d28f117d20f3b7f0d462267bd3c685cca8f4584532dfee0e8b9bacefa3092d28fcce7953a28f82e4ba6b3a1430ecca58b770dab656bed1b224663e196dffc28c96a2c65ef9de1989a125ecf2fed47eb96bef8a636a91bd521c47aeb8bc011bf81cc688fd8b620446353cbf7692201b5552cb07fb02eb3954dfaa6f5c31bf91e20b84419dcbbdaba0c31a124d8f4218b2f88da3eba44dbe40eb290052538dccd0ff7670de5f33a83ff74895b66adcff58c9c21e93b31bb49ccb2e026995ee155b5517b72daa76526a2e42aa6fa94357cd42e2a8a1d3e7d4cefc33d5d07d6303d798d2551a21f862b5f492d0c7cf078a77007a02847b34675dfad4fb457e9f20dc5750fb127a3c31b9d6a3996d50ac3ffc6ef29cca1d8414d0438bf3271dc4f4e00cfe19a507b447dc310f74aeb2a3c0b3fae6d7d13f4935bc72c35df3efa6e879164421505ee32d93b030e32a7970b53430b1643855167278e5058c4a48a7840e2fcdb282e45b5b86c0b2756f19b595f3bcfc926df35e33ac26dd1e88cd394015a5f54deb4c9f4a0bef0eabcb27c4eb88dc2302f09e92f1bcc4b4754df1eeb536154543c7dbf181c9979fe6ed08311e5a3acf365ebb5745212b2630e83b3a5bd5fa4834c727248b165700c7435f8cb6ee455bad16ee0da68fe6acd2062dae9c8bc178b157b29ade98a9bbbd4c723a3dcb7852c7978b488e4f73a2c9163dbdffae175119f812b6f4b70c2b498704bc2b58603f167f277a74e64ec296a6dfdb0de3486c0f36ac1b55f80af9fc817ba4f84b898b2a3c5725e2faf466bb26a8a84f91e123d182033a7ae2029236aa4b673ceb50c1733d7edd60e3f119b7141c882d508e0331689c96fbfb9f7e888fe88561de427c721123036737c1460b0da00d3f958b948f68fcb321ab4e297290f781ff8afb06b755d82a7e6ce1963761d799eed786524bf19801b4877b2d856becdf7e87d71aa359f2d51f09de64bcbf27d0c3aceac70790e314fd06c2f5216f3d10574b7302d6bc2775b185145c1b741524567c456d42c5826f93afa20ae7196ca7224c3b69b1eada9eee752fb6d43f24170fcc02af7e1dea73f0f884f936f900165800acb9d57480a31e409d3f676ed92b6812cf182a088fc49d68082aa19c7be0711f436db1d7be44d97dc9405591a8d3e7f6f731c6f3e6c401749829b7624497f5eeac1fc782e7d6988340541f2617a317e", "2a6283b1c02c6aaf74c4155091ff54a904bb700077f96a9c4bd84e8e51b54d01"); + TestSHA3_256("9271fd111dcf260c04cf4b748f269ac80f7485c41f7724352a7ed40b2e2125b0bf30f3984ee9d21aab6eb07ec976b557c2426e131ad32bd0485aa57172f0e4f1798760f8352067ac023fbeca7b9c8bf5851c724e90ffff44195b44ae73c9c317c85e8e585bddac6d0f2abf812d02e44b62eadb9d0765683aa56af8e9b91588c7b49dc3e146866a02dc18f9ca680f88006094ef29096c2d5af5700b4aca3dfcab462c48bb8085691671efb5ceb22b3ebd8702f71a1d7c184b1053c3fa30a7e76b85f3650d9140714fd4993bb496becf2ae01d3a98ccfdefb6fefd692173bd11af7adb61ffff214a550ffcd3a5993004ee72cb02ca9c577b42c85444e619e6411e2bca86bb548ebbd12a02c5c945eaa3b246f595d817f3849875429e72ac894160a2a91a6617f18e6b2b9258472152741d62843cebc537d25f0daebdedb410d71ee761662bd1b189ca1c93d648b5d141d8d05e3f2b2d8c2c40997fea7eb7e2cc0000d8b2300936759704ef85f38ad0d08a986de6bfd75b5be3209f6d4d3f67e7adf7f8469d47e81979ec8dae7127b5eadcc09779cf4b0a28efaaf58e83d307f2ced4a8699b142f3f19db5598e914e9577652c63f851203580d40699548fc2ab30a9dcf6452f673ad1ed92f8d84dad5dfff55e18107b3acb6e4e8e3c9c34038f40a5c577fe9771c2c31ef03d36a00e04a20d2d0877db66f091dac4b741d2a997b75182702881f9284fa23b9b3c20e715f80d07b9910a4b3185f9489dc7d3fb510f4da273559753d7d207f3975b48df2e7c857caffe703dfac53a786490c09f57d2fa93f60810186df4c0b6b616a04caab9f70a5002c5e5d8da0ed2805f20fbf89cd8d57ca2b4bd37125ce38bf09fb6170ae21f4e6043a9483ef6e585756d97cfb778e57bc7ddc8dfc54d086d6bcfa1f019c749ff79921ec56e833ff8660f0959cd4b5277e9f3b1d4880193fefa98a6c2512718e7c139acdcd324303db3adb70348d09b058baf0e91d52b24952f832b0a3b81fa9bc9a2e9fb276a64e9e0922778b4992d892f6845b4372a28e47d27b53443586d9015463cacb5b65c617f84e1168b15988737a7eda8187f1f4165fecbdd032ae04916cc4b6e18a87558d2ce6a5946c65a9446f66cda139a76506c60d560f56a013b508d6ccbbaa14e24ad0729dd823bf214efcc59e6932cdc860306687c84a63efb551237223641554940a7a60fa7e6ddad64a21b4a2176b046dc480b6c5b5ff7ed96e3211df609195b4028756c22479ba278105771493870372abe24dcc407daa69878b12b845908cf2e220e7fabeeaab88c8f64f864c2bacba0c14b2a693e45aacc6b7db76bc1a2195cfce7b68f3c99440477ea4c1ea5ee78c109f4f1b553c76eb513dd6e16c383ce7f3187ad66c1d5c982724de8e16299c2fde0a8af22e8de56e50a56ac0fef1c52e76864c0ad1eeedd8907065b37892b3eca0ddcdf5c8e0917dec78fedd194ea4b380a059ccc9452e48a9eba2f8b7a4150b7ba17feac83c61604c3cfcfe6655c2be37ef0ae6fc29072f9b1cfb277b64a8d499dd079ad9aa3d5e9a7ccbec8c100596c6fac51e13a260d78d8cd9066edc558e2219cfcda1310dc1fbbdd36f348756855349f33eb6b82186a8c1a55f361305833edd3e4ac8d9b9cf99897c4e06c19ed10765fd0c8c7433851445c5f87b119ef913b2bcdbf7aa2ad19c672e53a9c6c3c309d549513edd7c1cf8a0a399e6df0939cc1fb146d6ad460e2ce05144c69eafa3822141d473fbe5927c58a50c1e842f8b8fad85540ce9f6d06f7b4dea045248b999d24c5fd4d75631caf73518cc08f73684e2a1cd4266235d90c08a0d0ce8784c776fd1b80978b83f0705ba8498744884d5496b791f2db3ffb5377175856b25a643803aa8b9e7f1055e089c1929cf0cbba7674c204c4590fb076968e918e0390d268eeef78c2aebcbf58a429f28212a2425c6ad8970b6a09cadddd8336d519bca4820556d2c4b8cd9f41216de3c728a0774edf47d3489cd29cf1b2a192bc53325d0bed7d23e51be7684297f9d0ecb14acbf648bc440c5fde997acc464fb45e965e6f0dced6d4568ebcd55e5a64633b05a2cb4d8263b721a252b1710dc84d8a5d4b43fcc875e2e7281f621b0bf8bb3465be364456bcd990b26b3e474486f864fb85f320f68bc14c37d271249b18552bef50dfc385a9f41b831589c5a716357cf5a12520d582d00452a8ab21643dd180071d2041bbc5972099141c6292009540d02f3252f1f59f8dfcf4488803f3b0df41759055559a334e68c98ea491b0984f2f82a35db84ea0779b3801cf06b463a832e", "4e75bf3c580474575c96ec7faa03feb732379f95660b77149974133644f5d2a0"); + TestSHA3_256("075997f09ab1980a3179d4da78c2e914a1ff48f34e5d3c2ab157281ef1841052d0b45a228c3cd6b5028efd2d190d76205e1fdf4cec83c9868fe504f429af1e7c5423267c48a7b5bc005f30a1980147a3fae5c100b95c7cb23d43af9f21d87311d9cc826598993e077015f59ebc476383bb7a78787d915c97039ab188a2a618f7a8d7f64542ba787e9dd7d48c4c87d2aaea068c1b00c9711b2812901673c11418096d0a850fb36b0acece56d311689dfeceb0835009adc427f6d2d6b05ed26f5a43b6478bc72c1f914a2202dbd393cb69b1a1e78162e55ca4b3030ac0298131a7a0d934c032cc9dfc5afa600c59b064d2d9013f15d1184278a8ccb5ad9d7563e666fe5a8c173cec34467ef9cf6d6671208ff714741fee7c8d1d565edf82570dffde4f3f584024142056d8548ad55df83d1babed06141114c95ac88dbea0ce35d950f16d8a732a1ea7d22dfaa75a3e0410c546523277261116a64bcbb2be83e55e040f6c8c79f911b301a8718cc4b19a81d5f0cb6312d87c5b4b079e23a61d247541cfc2c41a37f52b2c6e43a3db5dc47892d0e1feabcc5c808f2391791e45fb065159f99c1d8dd2f69baaf75267eb89dd460f1b6c0badb96cbbc8291cefa370fa7ad6997a4ca2b1fe968216032f02f29837d40215fa219c09161df074e1de8e37056e28c86d1f992a651e271dfc4b0592ad481c613fd00c3eea4b6deabb9f5aa63a4830ed49ab93624fa7b208966eccb1f293f4b9a46411f37d7928e4478dde2f608d3851a8efa68e9d45402bc5124fde4ddc0f83ef82b31019d0aacb4b5121bbc064c95c5292da97981f58f051df9502054bf728e9d4fb7e04787a0890922b30a3f66a760e3d3763855e82be017fa603630a33115a02f02386982001def905784f6ba307a598c6dbaf2946fe9e978acbaf3e4ba50ab49ae8e9582520fc2eb6790deafc77e04a8ee75da92d16f0d249403112c74bc09102b573e110ccb4d8461d249bfe2e85fc9770d606be6fbfd5ec4c30ac306d46412f736e5b696ccc9fbe4adea730955c55ea5c63678271d34b7bd6f6340e72626d290820eeb96a0d2d25ea81361a122ffe8e954cf4ff84f4dafcc5c9d3e7c2ddbdf95ed2c0862d3f2783e4566f450ec49e8b01d9d7bf11e92a7903f2b045c57ed8a65ccbfc5b1d2a38e020a57b38f2e4deea8a52354a7e7be4f977b8f5afe30f6738e955c8bda295064586b6827b245766b217fe39263572b0850965c7ae845611b8efb64c36244a39b9fed0ab970ee5ddeb8f2608dd9c963524a14050c9101d7f2d5537b24d0b0f7a45703c1e131656ec9edc12cdf71dae1cde2790b888ef2a589f03201f8bbfad71f0c4430477a6713ad2e50aaefa1f840cbb839e277389454517e0b9bd76a8ecc5c2e22b854c25ff708f9256d3700adeaec49eb2c4134638ee9bd649b4982f931ec3b23cc819fbc835ddcb3d65e04585aa005e13b7ef8fcafa36cc1a2c79ba6c26fc1dc0f6668f9432c578088cd33a41a778ac0b298fcac212edab724c9fb33d827409fd36bc4b2b0e4e81006fd050d94d3271e0427c61e9ddca599a3c9480cfdd33603cb1a196557281ce6a375fef17463893db293dba0704d4bfda25e08beadd4208c58ea0d8d9066448910b087fc13792fc44075a3fe42e13c5792f093a552aa8ebe0f63e7a807102d5bc145468a0cb469263035c5647049054c18199f7da6d6defd51105e2125c605e327aca137ca85e3f7f46ca69f92d5252f84418293f4e9afeeb067c79576e88cc3c64f3e61d76e1e9e2f72cdfc35261a9679f0c374d7436ff6cfe2ba71650810522fa554a4aded87ad23f0b206b1bc63f56bbff8bcc8849d99e209bd519a953f32c667aa8cd874ad99846ed94b92f88fe0dbf788c8431dc76ca9553692622077da2cdea666c1b3fee7c335da37737afccd3d400a23d18f5bd3784dbcd0663a38acb5a2beef03fc0a1c52ee0b56bda4493f2221e35bee59f962f16bc6781133204f032c7a6209dd3dabd6100325ec14e3ab0d05aadd03fdfe9f8737da15edab9d2598046f8c6dd8381aaf244821994d5a956073c733bcebf9edbc2a6e2676242dc4e6a2e4ba8a7d57ed509340d61fae2c82bee4dedc73b469e202cc0916250d40a1718090690a1d3b986cf593b019b7b7f79ae14843b2e7ccf0fd85218184f7844fbb35e934476841b056b3a75bf20abb6866e19a0614e6a1af0eee4de510535724363b6598cccf08a99066021653177559c57e5aaff4417670a98fe4bd41a137c384f98c0324c20ef8bc851a9b975e9440191ff08deb78c9fa6fc29c76b371a4a1fa08c30fc9d1b3323d897738495086bfd43ef24c650cfa80c42ecbadc0453c4437d1a11b467e93ca95fbae98d38dcb2da953e657fb7ea6c8493d08cf028c5d3eb0fcbcb205493f4658440719e076e02deb07332d093e4d256175ca56f4c785d5e7e26c6090a20429f70b3757daac54153bc16f5828dc6c1c9f5186e2117754be5f1b46b3631980d9e4a9a5c", "2e07737d271b9a0162eb2f4be1be54887118c462317eb6bd9f9baf1e24111848"); + TestSHA3_256("119a356f8c0790bbd5e9f3b4c5c4a70e97f462364c88cad04d5435645342b35484e94e12df61908fd95546f74859849b817ee92fbd242435c210b7b9bfbffb3f77f965faa1a9073e8feb5a380f673add8fde32208402fa680c8b3e41d187a15131f1028f9d86feaf3fd4b6e0e094d2ba0839c67267c9535173ec51645343ad74fcfaae389aa17cca3137e2588488531c36ba2b8e2f2238d8415c798a0b9a258f1e3cef605fa18977ad3d6707c3ecc5ea5f86ebdaa4b4b0e5bc023d1bc335138ae0de506cb52f2d9efa0ecc546468310cccc88ec08d28c3602e07257f41bb7e4d8a0956c564f3712761d199a931a39e69c5a69aa7b3257931dd92b91e4ed56fbf64e48bd334945cfa2aaf576df04614eb914899f7df54db4012cc8261b12bedcab69876feedbbf7009dcf8d076af89b797ad71217d75cf07514dc07ae34640055c74c9faf560f491f015ac3e167623cfbc67b8e7163e7c1b92debd06e9d28b049e0298f4c38395a40a0778162af2cfe5abe5b946c4d9a54f2a321660ab521068c4957cd3f5be0324cc04f50f209fdea7caaa0ac705c1fb30abfa550e844f509074afde1ee87adda29aa09b7f93e7d064ad2715ee5571ee6e7c9a01672124cc2a22b4354c3844759c1a6ce3fdf17555cac7df73334073ef3730939410fe6cf37463352ad241958b7fafbc66e0d592df48bf55ab2c33428e494c6995826892572d9ab52747b1085fcbef318cfe9cfacd4cd80c164fba584c1344ae7e321c4f77b44db6b322f2f7831f4d4ede7dc407b065c6754cf4424f9903adb4c6b675ded58700bc36da83fd95e84e76c404f7342921ef23d7d07772f5b8ec2f077601cae13448385b04e074f895574be61a831a87efd68a1f6aa67cf291847fa8f74cbe3b4a78ad780895183bc51c30ad2514255d4e013abc097bc8103c0b1933b0b303341836ae167c1e31dfe5f1b791cb06ef29cae398065343eecf06e4ae2048d1547c4bf69ccec5e86c45867c633c62f7d27dc51234b6debb5b9f80a5810716240c64443d0c098c80220d0520a5f5834369b9eb019325e23e88f237c24440bf27959caf7e7e4f1671fda710630255a2962f7e9b3625dc243a0177aacf6a758a68aa85dc3f56181a4a59a406c7fae5575c9e2c64248f520b5a5f904821661e2e43a5a058f445fd0e55b07476c4122d18033053b45112201e0bfdcc9e7cb9931155018ca431a0564930aca8defbca954b2680753a4060bec2cb668d2c15e77cba29589b5c7c07bc7177a8b1adb3a6968732f9213476fd96901514626fa17243af1d156cd037eea81d773f1f71a018d942b524b851794b300c7591ecd783ec8066ccb261bdf9b7a183dbda42b92593b614297dcb0fabcc23ae69797d0251b8ab57a4da2a544615216b01f4dbe2d8c9b5520c7ed9cd9312e9ec6d05a36e7f693d1821d727518169b03976394b9d1e1d7fa2daa25529d391eb5d0cf0f07a8160be2ee043d9345037c655c4f2023689f14d8d2072dd92c1dba056a5b5d4c4fc4196e25caab05b1701ec666ac9a04d90f7d7575a7ac3970252c18fd3bec0cc448e5ff8f3765d546a4a8ad1d41a9640c79375b80534b7b50989976f238654fefea981c9413130beae943a3e9d8f64ce9256d1259d1b2a6b3c02ca5af1a701db8f25a4e9c255dad8785172f323728c3585a45206ae988c283e30a2f9ea9b47f07a7521b0f36e9c504c14bd96027e8d24161e70f196576d8a74a5e9c26acda7cc452a90e550e625a49e50829db70de808c827c67d00c23ee073d4e72aeed891dd73b86acd6756e753e3975a80cdab1d521052caef6a5380f8b03023ba0326a6928aa127ffb33b51dcb05bbdd592d0ad9e8321e6ef2f95c401be6a37e634425689fe7750e2a0fe05ad89001502b309095ca517b2e2ed0388b9f2c59c45feb61222539d6e1ccd397344d23708aebacec10ada96a7711f173f7ff1e4b94fceec6a0a0ea5d814a4581b412063012ff6ac5527b8314d00326b68c2304a276a217fde9fa4034750a2e47e10f816870d12fc4641a27a1c16c35a953f32685f2b92cae0519848045765591c42ddc402dc7c6914d74dd38d2b5e7f35358cb1d91a9f681fde7fd6c7af5840663525ee1d04bf6d3156fed018c44043d95383d92dada3d1cd84af51d9bee814ec8675073e1c48632c5f59e257682542e1f7dc20b56b5c92a9e2cb2be30cb1512fb55fa1de99a3f5864ed3acc19d79e6ffb0da3b08ba0615157747d75c1f308fa0202a4086f34e9eafa3e071dfbacaca731d228aa0304cf390c0a9e6ad7ce22ade758965cfbfc4c9390d24e41a667447fc7b29821464ad98bc5d65dc7f9c42bd4b23e174015592ff92c905660a2722f9fc7973d3cdad848ef88bf02b1b03dea16699b71dc46b35bc4d96069a0753335ae38685d244918e30c5fb0d45283a1281c1659ea591573999d9c2acd2ca9141d55230d41011b70748b518e1cd2fa58ad8dc05fcbdf0bffaf2c7fd6cb2ac67bb13b8f6d31fad64ac113664223599dca411270955c95aec06518894dabc352d2b70984727437040d944da7b42e0ef560ac532de3e4a4891e8509c275b51ed780f8660b0354e12c21b3e11bcc88198980b5f7ff31ad342182d5a933373164dced3cfb2a081720d7eee676cb7378a3e19326a7ee67fd6c00521f9de37c66bcea814b6feb6a061b8cdcf7b4bd8f45d48602c5", "c26d0b064e409df64819cd7c1a3b8076f19815b9823adac4e3ce0b4d3a29de18"); + TestSHA3_256("72c57c359e10684d0517e46653a02d18d29eff803eb009e4d5eb9e95add9ad1a4ac1f38a70296f3a369a16985ca3c957de2084cdc9bdd8994eb59b8815e0debad4ec1f001feac089820db8becdaf896aaf95721e8674e5d476b43bd2b873a7d135cd685f545b438210f9319e4dcd55986c85303c1ddf18dc746fe63a409df0a998ed376eb683e16c09e6e9018504152b3e7628ef350659fb716e058a5263a18823d2f2f6ee6a8091945a48ae1c5cb1694cf2c1fe76ef9177953afe8899cfa2b7fe0603bfa3180937dadfb66fbbdd119bbf8063338aa4a699075a3bfdbae8db7e5211d0917e9665a702fc9b0a0a901d08bea97654162d82a9f05622b060b634244779c33427eb7a29353a5f48b07cbefa72f3622ac5900bef77b71d6b314296f304c8426f451f32049b1f6af156a9dab702e8907d3cd72bb2c50493f4d593e731b285b70c803b74825b3524cda3205a8897106615260ac93c01c5ec14f5b11127783989d1824527e99e04f6a340e827b559f24db9292fcdd354838f9339a5fa1d7f6b2087f04835828b13463dd40927866f16ae33ed501ec0e6c4e63948768c5aeea3e4f6754985954bea7d61088c44430204ef491b74a64bde1358cecb2cad28ee6a3de5b752ff6a051104d88478653339457ac45ba44cbb65f54d1969d047cda746931d5e6a8b48e211416aefd5729f3d60b56b54e7f85aa2f42de3cb69419240c24e67139a11790a709edef2ac52cf35dd0a08af45926ebe9761f498ff83bfe263d6897ee97943a4b982fe3404ef0b4a45e06113c60340e0664f14799bf59cb4b3934b465fabefd87155905ee5309ba41e9e402973311831ea600b16437f71df39ee77130490c4d0227e5d1757fdc66af3ae6b9953053ed9aafca0160209858a7d4dd38fe10e0cb153672d08633ed6c54977aa0a6e67f9ff2f8c9d22dd7b21de08192960fd0e0da68d77c8d810db11dcaa61c725cd4092cbff76c8e1debd8d0361bb3f2e607911d45716f53067bdc0d89dd4889177765166a424e9fc0cb711201099dda213355e6639ac7eb86eca2ae0ab38b7f674f37ef8a6fcca1a6f52f55d9e1dcd631d2c3c82bba129172feb991d5af51afecd9d61a88b6832e4107480e392aed61a8644f551665ebff6b20953b635737a4f895e429fddcfe801f606fbda74b3bf6f5767d0fac14907fcfd0aa1d4c11b9e91b01d68052399b51a29f1ae6acd965109977c14a555cbcbd21ad8cb9f8853506d4bc21c01e62d61d7b21be1b923be54914e6b0a7ca84dd11f1159193e1184568a6134a6bbadf5b4df986edcf2019390ae841cfaa44435e28ce877d3dae4177992fa5d4e5c005876dbe3d1e63bec7dcc0942762b48b1ecc6c1a918409a8a72812a1e245c0c67be6e729c2b49bc6ee4d24a8f63e78e75db45655c26a9a78aff36fcd67117f26b8f654dca664b9f0e30681874cb749e1a692720078856286c2560b0292cc837933423147569350955c9571bf8941ba128fd339cb4268f46b94bc6ee203eb7026813706ea51c4f24c91866fc23a724bf2501327e6ae89c29f8db315dc28d2c7c719514036367e018f4835f63fdecd71f9bdced7132b6c4f8b13c69a517026fcd3622d67cb632320d5e7308f78f4b7cea11f6291b137851dc6cd6366f2785c71c3f237f81a7658b2a8d512b61e0ad5a4710b7b124151689fcb2116063fbff7e9115fed7b93de834970b838e49f8f8ba5f1f874c354078b5810a55ae289a56da563f1da6cd80a3757d6073fa55e016e45ac6cec1f69d871c92fd0ae9670c74249045e6b464787f9504128736309fed205f8df4d90e332908581298d9c75a3fa36ab0c3c9272e62de53ab290c803d67b696fd615c260a47bffad16746f18ba1a10a061bacbea9369693b3c042eec36bed289d7d12e52bca8aa1c2dff88ca7816498d25626d0f1e106ebb0b4a12138e00f3df5b1c2f49d98b1756e69b641b7c6353d99dbff050f4d76842c6cf1c2a4b062fc8e6336fa689b7c9d5c6b4ab8c15a5c20e514ff070a602d85ae52fa7810c22f8eeffd34a095b93342144f7a98d024216b3d68ed7bea047517bfcd83ec83febd1ba0e5858e2bdc1d8b1f7b0f89e90ccc432a3f930cb8209462e64556c5054c56ca2a85f16b32eb83a10459d13516faa4d23302b7607b9bd38dab2239ac9e9440c314433fdfb3ceadab4b4f87415ed6f240e017221f3b5f7ac196cdf54957bec42fe6893994b46de3d27dc7fb58ca88feb5b9e79cf20053d12530ac524337b22a3629bea52f40b06d3e2128f32060f9105847daed81d35f20e2002817434659baff64494c5b5c7f9216bfda38412a0f70511159dc73bb6bae1f8eaa0ef08d99bcb31f94f6be12c29c83df45926430b366c99fca3270c15fc4056398fdf3135b7779e3066a006961d1ac0ad1c83179ce39e87a96b722ec23aabc065badf3e188347a360772ca6a447abac7e6a44f0d4632d52926332e44a0a86bff5ce699fd063bdda3ffd4c41b53ded49fecec67f40599b934e16e3fd1bc063ad7026f8d71bfd4cbaf56599586774723194b692036f1b6bb242e2ffb9c600b5215b412764599476ce475c9e5b396fbcebd6be323dcf4d0048077400aac7500db41dc95fc7f7edbe7c9c2ec5ea89943fe13b42217eef530bbd023671509e12dfce4e1c1c82955d965e6a68aa66f6967dba48feda572db1f099d9a6dc4bc8edade852b5e824a06890dc48a6a6510ecaf8cf7620d757290e3166d431abecc624fa9ac2234d2eb783308ead45544910c633a94964b2ef5fbc409cb8835ac4147d384e12e0a5e13951f7de0ee13eafcb0ca0c04946d7804040c0a3cd088352424b097adb7aad1ca4495952f3e6c0158c02d2bcec33bfda69301434a84d9027ce02c0b9725dad118", "d894b86261436362e64241e61f6b3e6589daf64dc641f60570c4c0bf3b1f2ca3"); +} + BOOST_AUTO_TEST_SUITE_END() From f6f86af8ce228a1c81505186ec2c26fe2c78ad93 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 30 Aug 2020 16:47:35 -0700 Subject: [PATCH 25/58] Unroll Keccak-f implementation --- src/crypto/sha3.cpp | 99 ++++++++++++++++++++++++++++++--------------- 1 file changed, 66 insertions(+), 33 deletions(-) diff --git a/src/crypto/sha3.cpp b/src/crypto/sha3.cpp index 5f9ce451461c4..9c0c42fa7743f 100644 --- a/src/crypto/sha3.cpp +++ b/src/crypto/sha3.cpp @@ -30,48 +30,81 @@ void KeccakF(uint64_t (&st)[25]) 0x8000000000008002, 0x8000000000000080, 0x000000000000800a, 0x800000008000000a, 0x8000000080008081, 0x8000000000008080, 0x0000000080000001, 0x8000000080008008 }; - static constexpr int ROTC[24] = { - 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 2, 14, - 27, 41, 56, 8, 25, 43, 62, 18, 39, 61, 20, 44 - }; - static constexpr int PILN[24] = { - 10, 7, 11, 17, 18, 3, 5, 16, 8, 21, 24, 4, - 15, 23, 19, 13, 12, 2, 20, 14, 22, 9, 6, 1 - }; static constexpr int ROUNDS = 24; for (int round = 0; round < ROUNDS; ++round) { - uint64_t bc[5], t; + uint64_t bc0, bc1, bc2, bc3, bc4, t; // Theta - for (int i = 0; i < 5; i++) { - bc[i] = st[i] ^ st[i + 5] ^ st[i + 10] ^ st[i + 15] ^ st[i + 20]; - } - - for (int i = 0; i < 5; i++) { - t = bc[(i + 4) % 5] ^ Rotl(bc[(i + 1) % 5], 1); - for (int j = 0; j < 25; j += 5) st[j + i] ^= t; - } + bc0 = st[0] ^ st[5] ^ st[10] ^ st[15] ^ st[20]; + bc1 = st[1] ^ st[6] ^ st[11] ^ st[16] ^ st[21]; + bc2 = st[2] ^ st[7] ^ st[12] ^ st[17] ^ st[22]; + bc3 = st[3] ^ st[8] ^ st[13] ^ st[18] ^ st[23]; + bc4 = st[4] ^ st[9] ^ st[14] ^ st[19] ^ st[24]; + t = bc4 ^ Rotl(bc1, 1); st[0] ^= t; st[5] ^= t; st[10] ^= t; st[15] ^= t; st[20] ^= t; + t = bc0 ^ Rotl(bc2, 1); st[1] ^= t; st[6] ^= t; st[11] ^= t; st[16] ^= t; st[21] ^= t; + t = bc1 ^ Rotl(bc3, 1); st[2] ^= t; st[7] ^= t; st[12] ^= t; st[17] ^= t; st[22] ^= t; + t = bc2 ^ Rotl(bc4, 1); st[3] ^= t; st[8] ^= t; st[13] ^= t; st[18] ^= t; st[23] ^= t; + t = bc3 ^ Rotl(bc0, 1); st[4] ^= t; st[9] ^= t; st[14] ^= t; st[19] ^= t; st[24] ^= t; // Rho Pi t = st[1]; - for (int i = 0; i < 24; i++) { - int j = PILN[i]; - bc[0] = st[j]; - st[j] = Rotl(t, ROTC[i]); - t = bc[0]; - } - - // Chi - for (int j = 0; j < 25; j += 5) { - for (int i = 0; i < 5; i++) bc[i] = st[j + i]; - for (int i = 0; i < 5; i++) { - st[j + i] ^= (~bc[(i + 1) % 5]) & bc[(i + 2) % 5]; - } - } + bc0 = st[10]; st[10] = Rotl(t, 1); t = bc0; + bc0 = st[7]; st[7] = Rotl(t, 3); t = bc0; + bc0 = st[11]; st[11] = Rotl(t, 6); t = bc0; + bc0 = st[17]; st[17] = Rotl(t, 10); t = bc0; + bc0 = st[18]; st[18] = Rotl(t, 15); t = bc0; + bc0 = st[3]; st[3] = Rotl(t, 21); t = bc0; + bc0 = st[5]; st[5] = Rotl(t, 28); t = bc0; + bc0 = st[16]; st[16] = Rotl(t, 36); t = bc0; + bc0 = st[8]; st[8] = Rotl(t, 45); t = bc0; + bc0 = st[21]; st[21] = Rotl(t, 55); t = bc0; + bc0 = st[24]; st[24] = Rotl(t, 2); t = bc0; + bc0 = st[4]; st[4] = Rotl(t, 14); t = bc0; + bc0 = st[15]; st[15] = Rotl(t, 27); t = bc0; + bc0 = st[23]; st[23] = Rotl(t, 41); t = bc0; + bc0 = st[19]; st[19] = Rotl(t, 56); t = bc0; + bc0 = st[13]; st[13] = Rotl(t, 8); t = bc0; + bc0 = st[12]; st[12] = Rotl(t, 25); t = bc0; + bc0 = st[2]; st[2] = Rotl(t, 43); t = bc0; + bc0 = st[20]; st[20] = Rotl(t, 62); t = bc0; + bc0 = st[14]; st[14] = Rotl(t, 18); t = bc0; + bc0 = st[22]; st[22] = Rotl(t, 39); t = bc0; + bc0 = st[9]; st[9] = Rotl(t, 61); t = bc0; + bc0 = st[6]; st[6] = Rotl(t, 20); t = bc0; + st[1] = Rotl(t, 44); - // Iota - st[0] ^= RNDC[round]; + // Chi Iota + bc0 = st[0]; bc1 = st[1]; bc2 = st[2]; bc3 = st[3]; bc4 = st[4]; + st[0] = bc0 ^ (~bc1 & bc2) ^ RNDC[round]; + st[1] = bc1 ^ (~bc2 & bc3); + st[2] = bc2 ^ (~bc3 & bc4); + st[3] = bc3 ^ (~bc4 & bc0); + st[4] = bc4 ^ (~bc0 & bc1); + bc0 = st[5]; bc1 = st[6]; bc2 = st[7]; bc3 = st[8]; bc4 = st[9]; + st[5] = bc0 ^ (~bc1 & bc2); + st[6] = bc1 ^ (~bc2 & bc3); + st[7] = bc2 ^ (~bc3 & bc4); + st[8] = bc3 ^ (~bc4 & bc0); + st[9] = bc4 ^ (~bc0 & bc1); + bc0 = st[10]; bc1 = st[11]; bc2 = st[12]; bc3 = st[13]; bc4 = st[14]; + st[10] = bc0 ^ (~bc1 & bc2); + st[11] = bc1 ^ (~bc2 & bc3); + st[12] = bc2 ^ (~bc3 & bc4); + st[13] = bc3 ^ (~bc4 & bc0); + st[14] = bc4 ^ (~bc0 & bc1); + bc0 = st[15]; bc1 = st[16]; bc2 = st[17]; bc3 = st[18]; bc4 = st[19]; + st[15] = bc0 ^ (~bc1 & bc2); + st[16] = bc1 ^ (~bc2 & bc3); + st[17] = bc2 ^ (~bc3 & bc4); + st[18] = bc3 ^ (~bc4 & bc0); + st[19] = bc4 ^ (~bc0 & bc1); + bc0 = st[20]; bc1 = st[21]; bc2 = st[22]; bc3 = st[23]; bc4 = st[24]; + st[20] = bc0 ^ (~bc1 & bc2); + st[21] = bc1 ^ (~bc2 & bc3); + st[22] = bc2 ^ (~bc3 & bc4); + st[23] = bc3 ^ (~bc4 & bc0); + st[24] = bc4 ^ (~bc0 & bc1); } } From d41adb4ef41d463a97002f52f4d74f9ed9007bc4 Mon Sep 17 00:00:00 2001 From: Vasil Dimov Date: Mon, 31 Aug 2020 10:39:00 +0200 Subject: [PATCH 26/58] util: move HasPrefix() so it can be reused Move the function `HasPrefix()` from `netaddress.cpp` to `util/string.h` so it can be reused by `CNetAddr` methods (and possibly others). --- src/netaddress.cpp | 12 +++--------- src/util/string.h | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 1b67dad75658a..c815455ce3144 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -6,9 +6,10 @@ #include "netaddress.h" #include "hash.h" -#include "utilstrencodings.h" -#include "util/asmap.h" #include "tinyformat.h" +#include "util/asmap.h" +#include "util/string.h" +#include "utilstrencodings.h" #include #include @@ -50,13 +51,6 @@ void CNetAddr::SetIP(const CNetAddr& ipIn) m_addr = ipIn.m_addr; } -template -inline bool HasPrefix(const T1& obj, const std::array& prefix) -{ - return obj.size() >= PREFIX_LEN && - std::equal(std::begin(prefix), std::end(prefix), std::begin(obj)); -} - void CNetAddr::SetLegacyIPv6(Span ipv6) { assert(ipv6.size() == ADDR_IPV6_SIZE); diff --git a/src/util/string.h b/src/util/string.h index d07cb04af0adc..5a0c032df1140 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -5,6 +5,10 @@ #ifndef BITCOIN_UTIL_STRING_H #define BITCOIN_UTIL_STRING_H +#include "attributes.h" + +#include +#include #include #include #include @@ -41,4 +45,15 @@ inline bool ValidAsCString(const std::string& str) noexcept return str.size() == strlen(str.c_str()); } +/** + * Check whether a container begins with the given prefix. + */ +template +NODISCARD inline bool HasPrefix(const T1& obj, + const std::array& prefix) +{ + return obj.size() >= PREFIX_LEN && + std::equal(std::begin(prefix), std::end(prefix), std::begin(obj)); +} + #endif // BITCOIN_UTIL_STRENCODINGS_H From 2455420789df56711ab339ac56e0311879ef73ea Mon Sep 17 00:00:00 2001 From: furszy Date: Thu, 10 Jun 2021 17:34:53 -0300 Subject: [PATCH 27/58] test: move HasReason so it can be reused --- src/test/miner_tests.cpp | 15 --------------- src/test/test_pivx.h | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index a39fe6af9f30c..b3858a801fbe5 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -22,21 +22,6 @@ // future: this should be MAINNET. BOOST_FIXTURE_TEST_SUITE(miner_tests, WalletRegTestingSetup) -// BOOST_CHECK_EXCEPTION predicates to check the specific validation error -class HasReason { -public: - HasReason(const std::string& reason) : m_reason(reason) {} - bool operator() (const std::runtime_error& e) const { - bool ret = std::string(e.what()).find(m_reason) != std::string::npos; - if (!ret) { - std::cout << "error: " << e.what() << std::endl; - } - return ret; - }; -private: - const std::string m_reason; -}; - static struct { unsigned char extranonce; diff --git a/src/test/test_pivx.h b/src/test/test_pivx.h index e2cbb56a543ff..62463e68d556e 100644 --- a/src/test/test_pivx.h +++ b/src/test/test_pivx.h @@ -144,4 +144,19 @@ struct TestMemPoolEntryHelper // define an implicit conversion here so that uint256 may be used directly in BOOST_CHECK_* std::ostream& operator<<(std::ostream& os, const uint256& num); +// BOOST_CHECK_EXCEPTION predicates to check the specific validation error +class HasReason { +public: + HasReason(const std::string& reason) : m_reason(reason) {} + bool operator() (const std::runtime_error& e) const { + bool ret = std::string(e.what()).find(m_reason) != std::string::npos; + if (!ret) { + std::cout << "error: " << e.what() << std::endl; + } + return ret; + }; +private: + const std::string m_reason; +}; + #endif From 1f67e305ad3cd80c37c3930bf6f4e5f951ec0554 Mon Sep 17 00:00:00 2001 From: Vasil Dimov Date: Tue, 19 May 2020 15:01:50 +0200 Subject: [PATCH 28/58] net: CNetAddr: add support to (un)serialize as ADDRv2 Co-authored-by: Carl Dong --- src/netaddress.cpp | 55 ++++++++++++ src/netaddress.h | 133 +++++++++++++++++++++++++++- src/test/net_tests.cpp | 194 ++++++++++++++++++++++++++++++++++++++++- src/version.h | 3 + 4 files changed, 382 insertions(+), 3 deletions(-) diff --git a/src/netaddress.cpp b/src/netaddress.cpp index c815455ce3144..a29e140684262 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -14,10 +14,65 @@ #include #include #include +#include #include #include constexpr size_t CNetAddr::V1_SERIALIZATION_SIZE; +constexpr size_t CNetAddr::MAX_ADDRV2_SIZE; + +CNetAddr::BIP155Network CNetAddr::GetBIP155Network() const +{ + switch (m_net) { + case NET_IPV4: + return BIP155Network::IPV4; + case NET_IPV6: + return BIP155Network::IPV6; + case NET_ONION: + return BIP155Network::TORV2; + case NET_INTERNAL: // should have been handled before calling this function + case NET_UNROUTABLE: // m_net is never and should not be set to NET_UNROUTABLE + case NET_MAX: // m_net is never and should not be set to NET_MAX + assert(false); + } // no default case, so the compiler can warn about missing cases + + assert(false); +} + +bool CNetAddr::SetNetFromBIP155Network(uint8_t possible_bip155_net, size_t address_size) +{ + switch (possible_bip155_net) { + case BIP155Network::IPV4: + if (address_size == ADDR_IPV4_SIZE) { + m_net = NET_IPV4; + return true; + } + throw std::ios_base::failure( + strprintf("BIP155 IPv4 address with length %u (should be %u)", address_size, + ADDR_IPV4_SIZE)); + case BIP155Network::IPV6: + if (address_size == ADDR_IPV6_SIZE) { + m_net = NET_IPV6; + return true; + } + throw std::ios_base::failure( + strprintf("BIP155 IPv6 address with length %u (should be %u)", address_size, + ADDR_IPV6_SIZE)); + case BIP155Network::TORV2: + if (address_size == ADDR_TORV2_SIZE) { + m_net = NET_ONION; + return true; + } + throw std::ios_base::failure( + strprintf("BIP155 TORv2 address with length %u (should be %u)", address_size, + ADDR_TORV2_SIZE)); + } + + // Don't throw on addresses with unknown network ids (maybe from the future). + // Instead silently drop them and have the unserialization code consume + // subsequent ones which may be known to us. + return false; +} /** * Construct an unspecified IPv6 network address (::/128). diff --git a/src/netaddress.h b/src/netaddress.h index 6820c1cb053be..19d8f1f4435a4 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -14,12 +14,23 @@ #include "serialize.h" #include "span.h" #include "prevector.h" +#include "tinyformat.h" +#include "utilstrencodings.h" +#include "util/string.h" #include #include +#include #include #include +/** + * A flag that is ORed into the protocol version to designate that addresses + * should be serialized in (unserialized from) v2 format (BIP155). + * Make sure that this does not collide with any of the values in `version.h` + */ +static const int ADDRV2_FORMAT = 0x20000000; + /** * A network type. * @note An address may belong to more than one network, for example `10.0.0.1` @@ -182,7 +193,11 @@ class CNetAddr template void Serialize(Stream& s) const { - SerializeV1Stream(s); + if (s.GetVersion() & ADDRV2_FORMAT) { + SerializeV2Stream(s); + } else { + SerializeV1Stream(s); + } } /** @@ -191,17 +206,53 @@ class CNetAddr template void Unserialize(Stream& s) { - UnserializeV1Stream(s); + if (s.GetVersion() & ADDRV2_FORMAT) { + UnserializeV2Stream(s); + } else { + UnserializeV1Stream(s); + } } friend class CSubNet; private: + /** + * BIP155 network ids recognized by this software. + */ + enum BIP155Network : uint8_t { + IPV4 = 1, + IPV6 = 2, + TORV2 = 3, + }; + /** * Size of CNetAddr when serialized as ADDRv1 (pre-BIP155) (in bytes). */ static constexpr size_t V1_SERIALIZATION_SIZE = ADDR_IPV6_SIZE; + /** + * Maximum size of an address as defined in BIP155 (in bytes). + * This is only the size of the address, not the entire CNetAddr object + * when serialized. + */ + static constexpr size_t MAX_ADDRV2_SIZE = 512; + + /** + * Get the BIP155 network id of this address. + * Must not be called for IsInternal() objects. + * @returns BIP155 network id + */ + BIP155Network GetBIP155Network() const; + + /** + * Set `m_net` from the provided BIP155 network id and size after validation. + * @retval true the network was recognized, is valid and `m_net` was set + * @retval false not recognised (from future?) and should be silently ignored + * @throws std::ios_base::failure if the network is one of the BIP155 founding + * networks recognized by this software (id 1..3) and has wrong address size. + */ + bool SetNetFromBIP155Network(uint8_t possible_bip155_net, size_t address_size); + /** * Serialize in pre-ADDRv2/BIP155 format to an array. * Some addresses (e.g. TORv3) cannot be serialized in pre-BIP155 format. @@ -255,6 +306,25 @@ friend class CSubNet; s << serialized; } + /** + * Serialize as ADDRv2 / BIP155. + */ + template + void SerializeV2Stream(Stream& s) const + { + if (IsInternal()) { + // Serialize NET_INTERNAL as embedded in IPv6. We need to + // serialize such addresses from addrman. + s << static_cast(BIP155Network::IPV6); + s << COMPACTSIZE(ADDR_IPV6_SIZE); + SerializeV1Stream(s); + return; + } + + s << static_cast(GetBIP155Network()); + s << m_addr; + } + /** * Unserialize from a pre-ADDRv2/BIP155 format from an array. */ @@ -277,6 +347,65 @@ friend class CSubNet; UnserializeV1Array(serialized); } + + /** + * Unserialize from a ADDRv2 / BIP155 format. + */ + template + void UnserializeV2Stream(Stream& s) + { + uint8_t bip155_net; + s >> bip155_net; + + size_t address_size; + s >> COMPACTSIZE(address_size); + + if (address_size > MAX_ADDRV2_SIZE) { + throw std::ios_base::failure(strprintf( + "Address too long: %u > %u", address_size, MAX_ADDRV2_SIZE)); + } + + scopeId = 0; + + if (SetNetFromBIP155Network(bip155_net, address_size)) { + m_addr.resize(address_size); + s >> MakeSpan(m_addr); + + if (m_net != NET_IPV6) { + return; + } + + // Do some special checks on IPv6 addresses. + + // Recognize NET_INTERNAL embedded in IPv6, such addresses are not + // gossiped but could be coming from addrman, when unserializing from + // disk. + if (HasPrefix(m_addr, INTERNAL_IN_IPV6_PREFIX)) { + m_net = NET_INTERNAL; + memmove(m_addr.data(), m_addr.data() + INTERNAL_IN_IPV6_PREFIX.size(), + ADDR_INTERNAL_SIZE); + m_addr.resize(ADDR_INTERNAL_SIZE); + return; + } + + if (!HasPrefix(m_addr, IPV4_IN_IPV6_PREFIX) && + !HasPrefix(m_addr, TORV2_IN_IPV6_PREFIX)) { + return; + } + + // IPv4 and TORv2 are not supposed to be embedded in IPv6 (like in V1 + // encoding). Unserialize as !IsValid(), thus ignoring them. + } else { + // If we receive an unknown BIP155 network id (from the future?) then + // ignore the address - unserialize as !IsValid(). + s.ignore(address_size); + } + + // Mimic a default-constructed CNetAddr object which is !IsValid() and thus + // will not be gossiped, but continue reading next addresses from the stream. + m_net = NET_IPV6; + m_addr.assign(ADDR_IPV6_SIZE, 0x0); + } }; class CSubNet diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index c8e8fe1675fea..370755e701471 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -8,6 +8,7 @@ #include "net.h" #include "netbase.h" #include "serialize.h" +#include "span.h" #include "streams.h" #include "version.h" @@ -15,6 +16,7 @@ #include "test/test_pivx.h" +#include #include #include @@ -252,17 +254,207 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) BOOST_CHECK_EQUAL(addr.ToString(), "esffpvrt3wpeaygy.internal"); } -BOOST_AUTO_TEST_CASE(cnetaddr_serialize) +BOOST_AUTO_TEST_CASE(cnetaddr_serialize_v1) { CNetAddr addr; CDataStream s(SER_NETWORK, PROTOCOL_VERSION); + s << addr; + BOOST_CHECK_EQUAL(HexStr(s), "00000000000000000000000000000000"); + s.clear(); + + BOOST_REQUIRE(LookupHost("1.2.3.4", addr, false)); + s << addr; + BOOST_CHECK_EQUAL(HexStr(s), "00000000000000000000ffff01020304"); + s.clear(); + + BOOST_REQUIRE(LookupHost("1a1b:2a2b:3a3b:4a4b:5a5b:6a6b:7a7b:8a8b", addr, false)); + s << addr; + BOOST_CHECK_EQUAL(HexStr(s), "1a1b2a2b3a3b4a4b5a5b6a6b7a7b8a8b"); + s.clear(); + + BOOST_REQUIRE(addr.SetSpecial("6hzph5hv6337r6p2.onion")); + s << addr; + BOOST_CHECK_EQUAL(HexStr(s), "fd87d87eeb43f1f2f3f4f5f6f7f8f9fa"); + s.clear(); + addr.SetInternal("a"); s << addr; BOOST_CHECK_EQUAL(HexStr(s), "fd6b88c08724ca978112ca1bbdcafac2"); s.clear(); } +BOOST_AUTO_TEST_CASE(cnetaddr_serialize_v2) +{ + CNetAddr addr; + CDataStream s(SER_NETWORK, PROTOCOL_VERSION); + // Add ADDRV2_FORMAT to the version so that the CNetAddr + // serialize method produces an address in v2 format. + s.SetVersion(s.GetVersion() | ADDRV2_FORMAT); + + s << addr; + BOOST_CHECK_EQUAL(HexStr(s), "021000000000000000000000000000000000"); + s.clear(); + + BOOST_REQUIRE(LookupHost("1.2.3.4", addr, false)); + s << addr; + BOOST_CHECK_EQUAL(HexStr(s), "010401020304"); + s.clear(); + + BOOST_REQUIRE(LookupHost("1a1b:2a2b:3a3b:4a4b:5a5b:6a6b:7a7b:8a8b", addr, false)); + s << addr; + BOOST_CHECK_EQUAL(HexStr(s), "02101a1b2a2b3a3b4a4b5a5b6a6b7a7b8a8b"); + s.clear(); + + BOOST_REQUIRE(addr.SetSpecial("6hzph5hv6337r6p2.onion")); + s << addr; + BOOST_CHECK_EQUAL(HexStr(s), "030af1f2f3f4f5f6f7f8f9fa"); + s.clear(); + + BOOST_REQUIRE(addr.SetInternal("a")); + s << addr; + BOOST_CHECK_EQUAL(HexStr(s), "0210fd6b88c08724ca978112ca1bbdcafac2"); + s.clear(); +} + +BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) +{ + CNetAddr addr; + CDataStream s(SER_NETWORK, PROTOCOL_VERSION); + // Add ADDRV2_FORMAT to the version so that the CNetAddr + // unserialize method expects an address in v2 format. + s.SetVersion(s.GetVersion() | ADDRV2_FORMAT); + + // Valid IPv4. + s << MakeSpan(ParseHex("01" // network type (IPv4) + "04" // address length + "01020304")); // address + s >> addr; + BOOST_CHECK(addr.IsValid()); + BOOST_CHECK(addr.IsIPv4()); + BOOST_CHECK_EQUAL(addr.ToString(), "1.2.3.4"); + BOOST_REQUIRE(s.empty()); + + // Invalid IPv4, valid length but address itself is shorter. + s << MakeSpan(ParseHex("01" // network type (IPv4) + "04" // address length + "0102")); // address + BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure, HasReason("end of data")); + BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. + s.clear(); + + // Invalid IPv4, with bogus length. + s << MakeSpan(ParseHex("01" // network type (IPv4) + "05" // address length + "01020304")); // address + BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure, + HasReason("BIP155 IPv4 address with length 5 (should be 4)")); + BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. + s.clear(); + + // Invalid IPv4, with extreme length. + s << MakeSpan(ParseHex("01" // network type (IPv4) + "fd0102" // address length (513 as CompactSize) + "01020304")); // address + BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure, + HasReason("Address too long: 513 > 512")); + BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. + s.clear(); + + // Valid IPv6. + s << MakeSpan(ParseHex("02" // network type (IPv6) + "10" // address length + "0102030405060708090a0b0c0d0e0f10")); // address + s >> addr; + BOOST_CHECK(addr.IsValid()); + BOOST_CHECK(addr.IsIPv6()); + BOOST_CHECK_EQUAL(addr.ToString(), "102:304:506:708:90a:b0c:d0e:f10"); + BOOST_REQUIRE(s.empty()); + + // Valid IPv6, contains embedded "internal". + s << MakeSpan(ParseHex( + "02" // network type (IPv6) + "10" // address length + "fd6b88c08724ca978112ca1bbdcafac2")); // address: 0xfd + sha256("bitcoin")[0:5] + + // sha256(name)[0:10] + s >> addr; + BOOST_CHECK(addr.IsInternal()); + BOOST_CHECK_EQUAL(addr.ToString(), "zklycewkdo64v6wc.internal"); + BOOST_REQUIRE(s.empty()); + + // Invalid IPv6, with bogus length. + s << MakeSpan(ParseHex("02" // network type (IPv6) + "04" // address length + "00")); // address + BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure, + HasReason("BIP155 IPv6 address with length 4 (should be 16)")); + BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. + s.clear(); + + // Invalid IPv6, contains embedded IPv4. + s << MakeSpan(ParseHex("02" // network type (IPv6) + "10" // address length + "00000000000000000000ffff01020304")); // address + s >> addr; + BOOST_CHECK(!addr.IsValid()); + BOOST_REQUIRE(s.empty()); + + // Invalid IPv6, contains embedded TORv2. + s << MakeSpan(ParseHex("02" // network type (IPv6) + "10" // address length + "fd87d87eeb430102030405060708090a")); // address + s >> addr; + BOOST_CHECK(!addr.IsValid()); + BOOST_REQUIRE(s.empty()); + + // Valid TORv2. + s << MakeSpan(ParseHex("03" // network type (TORv2) + "0a" // address length + "f1f2f3f4f5f6f7f8f9fa")); // address + s >> addr; + BOOST_CHECK(addr.IsValid()); + BOOST_CHECK(addr.IsTor()); + BOOST_CHECK_EQUAL(addr.ToString(), "6hzph5hv6337r6p2.onion"); + BOOST_REQUIRE(s.empty()); + + // Invalid TORv2, with bogus length. + s << MakeSpan(ParseHex("03" // network type (TORv2) + "07" // address length + "00")); // address + BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure, + HasReason("BIP155 TORv2 address with length 7 (should be 10)")); + BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. + s.clear(); + + // Unknown, with extreme length. + s << MakeSpan(ParseHex("aa" // network type (unknown) + "fe00000002" // address length (CompactSize's MAX_SIZE) + "01020304050607" // address + )); + BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure, + HasReason("Address too long: 33554432 > 512")); + BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. + s.clear(); + + // Unknown, with reasonable length. + s << MakeSpan(ParseHex("aa" // network type (unknown) + "04" // address length + "01020304" // address + )); + s >> addr; + BOOST_CHECK(!addr.IsValid()); + BOOST_REQUIRE(s.empty()); + + // Unknown, with zero length. + s << MakeSpan(ParseHex("aa" // network type (unknown) + "00" // address length + "" // address + )); + s >> addr; + BOOST_CHECK(!addr.IsValid()); + BOOST_REQUIRE(s.empty()); +} + // prior to PR #14728, this test triggers an undefined behavior BOOST_AUTO_TEST_CASE(ipv4_peer_with_ipv6_addrMe_test) { diff --git a/src/version.h b/src/version.h index d40a760110a65..030dd9554eb14 100644 --- a/src/version.h +++ b/src/version.h @@ -43,4 +43,7 @@ static const int MEMPOOL_GD_VERSION = 60002; static const int NO_BLOOM_VERSION = 70005; +// Make sure that none of the values above collide with +// `ADDRV2_FORMAT`. + #endif // BITCOIN_VERSION_H From 8e508531909ce9c3e79f1895e8f44d816b0dee67 Mon Sep 17 00:00:00 2001 From: Sebastian Falbesoner Date: Fri, 7 Aug 2020 19:55:51 +0200 Subject: [PATCH 29/58] util: make EncodeBase32 consume Spans --- src/netaddress.cpp | 4 ++-- src/utilstrencodings.cpp | 8 ++++---- src/utilstrencodings.h | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/netaddress.cpp b/src/netaddress.cpp index a29e140684262..af2968dc3f9b8 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -372,9 +372,9 @@ enum Network CNetAddr::GetNetwork() const std::string CNetAddr::ToStringIP() const { if (IsTor()) - return EncodeBase32(m_addr.data(), m_addr.size()) + ".onion"; + return EncodeBase32(m_addr) + ".onion"; if (IsInternal()) - return EncodeBase32(m_addr.data(), m_addr.size()) + ".internal"; + return EncodeBase32(m_addr) + ".internal"; CService serv(*this, 0); struct sockaddr_storage sockaddr; socklen_t socklen = sizeof(sockaddr); diff --git a/src/utilstrencodings.cpp b/src/utilstrencodings.cpp index 7547af04b472f..7ce40d44a1ab5 100644 --- a/src/utilstrencodings.cpp +++ b/src/utilstrencodings.cpp @@ -192,20 +192,20 @@ std::string DecodeBase64(const std::string& str) return std::string((const char*)vchRet.data(), vchRet.size()); } -std::string EncodeBase32(const unsigned char* pch, size_t len) +std::string EncodeBase32(Span input) { static const char *pbase32 = "abcdefghijklmnopqrstuvwxyz234567"; std::string str; - str.reserve(((len + 4) / 5) * 8); - ConvertBits<8, 5, true>([&](int v) { str += pbase32[v]; }, pch, pch + len); + str.reserve(((input.size() + 4) / 5) * 8); + ConvertBits<8, 5, true>([&](int v) { str += pbase32[v]; }, input.begin(), input.end()); while (str.size() % 8) str += '='; return str; } std::string EncodeBase32(const std::string& str) { - return EncodeBase32((const unsigned char*)str.c_str(), str.size()); + return EncodeBase32(MakeUCharSpan(str)); } std::vector DecodeBase32(const char* p, bool* pfInvalid) diff --git a/src/utilstrencodings.h b/src/utilstrencodings.h index 9ca81339aa611..022cfb5d8356e 100644 --- a/src/utilstrencodings.h +++ b/src/utilstrencodings.h @@ -59,7 +59,7 @@ std::string EncodeBase64(Span input); std::string EncodeBase64(const std::string& str); std::vector DecodeBase32(const char* p, bool* pfInvalid = NULL); std::string DecodeBase32(const std::string& str); -std::string EncodeBase32(const unsigned char* pch, size_t len); +std::string EncodeBase32(Span input); std::string EncodeBase32(const std::string& str); std::string i64tostr(int64_t n); From a237ba4230d3aa293d7fa504d7ebab5792fd00ff Mon Sep 17 00:00:00 2001 From: Vasil Dimov Date: Thu, 27 Aug 2020 11:03:21 +0200 Subject: [PATCH 30/58] net: recognize TORv3/I2P/CJDNS networks Recognizing addresses from those networks allows us to accept and gossip them, even though we don't know how to connect to them (yet). Co-authored-by: eriknylund --- src/crypto/common.h | 7 ++ src/netaddress.cpp | 243 +++++++++++++++++++++++++++++++------- src/netaddress.h | 37 +++++- src/test/base32_tests.cpp | 5 +- src/test/net_tests.cpp | 104 +++++++++++++++- src/utilstrencodings.cpp | 13 +- src/utilstrencodings.h | 16 ++- 7 files changed, 367 insertions(+), 58 deletions(-) diff --git a/src/crypto/common.h b/src/crypto/common.h index 6906ee7807b27..7c4e09939c8d9 100644 --- a/src/crypto/common.h +++ b/src/crypto/common.h @@ -53,6 +53,13 @@ void static inline WriteLE64(unsigned char* ptr, uint64_t x) memcpy(ptr, (char*)&v, 8); } +uint16_t static inline ReadBE16(const unsigned char* ptr) +{ + uint16_t x; + memcpy((char*)&x, ptr, 2); + return be16toh(x); +} + uint32_t static inline ReadBE32(const unsigned char* ptr) { uint32_t x; diff --git a/src/netaddress.cpp b/src/netaddress.cpp index af2968dc3f9b8..dcb93e0c51dfd 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -4,8 +4,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "crypto/common.h" +#include "crypto/sha3.h" #include "netaddress.h" #include "hash.h" +#include "prevector.h" #include "tinyformat.h" #include "util/asmap.h" #include "util/string.h" @@ -29,7 +32,18 @@ CNetAddr::BIP155Network CNetAddr::GetBIP155Network() const case NET_IPV6: return BIP155Network::IPV6; case NET_ONION: - return BIP155Network::TORV2; + switch (m_addr.size()) { + case ADDR_TORV2_SIZE: + return BIP155Network::TORV2; + case ADDR_TORV3_SIZE: + return BIP155Network::TORV3; + default: + assert(false); + } + case NET_I2P: + return BIP155Network::I2P; + case NET_CJDNS: + return BIP155Network::CJDNS; case NET_INTERNAL: // should have been handled before calling this function case NET_UNROUTABLE: // m_net is never and should not be set to NET_UNROUTABLE case NET_MAX: // m_net is never and should not be set to NET_MAX @@ -66,6 +80,30 @@ bool CNetAddr::SetNetFromBIP155Network(uint8_t possible_bip155_net, size_t addre throw std::ios_base::failure( strprintf("BIP155 TORv2 address with length %u (should be %u)", address_size, ADDR_TORV2_SIZE)); + case BIP155Network::TORV3: + if (address_size == ADDR_TORV3_SIZE) { + m_net = NET_ONION; + return true; + } + throw std::ios_base::failure( + strprintf("BIP155 TORv3 address with length %u (should be %u)", address_size, + ADDR_TORV3_SIZE)); + case BIP155Network::I2P: + if (address_size == ADDR_I2P_SIZE) { + m_net = NET_I2P; + return true; + } + throw std::ios_base::failure( + strprintf("BIP155 I2P address with length %u (should be %u)", address_size, + ADDR_I2P_SIZE)); + case BIP155Network::CJDNS: + if (address_size == ADDR_CJDNS_SIZE) { + m_net = NET_CJDNS; + return true; + } + throw std::ios_base::failure( + strprintf("BIP155 CJDNS address with length %u (should be %u)", address_size, + ADDR_CJDNS_SIZE)); } // Don't throw on addresses with unknown network ids (maybe from the future). @@ -92,7 +130,13 @@ void CNetAddr::SetIP(const CNetAddr& ipIn) assert(ipIn.m_addr.size() == ADDR_IPV6_SIZE); break; case NET_ONION: - assert(ipIn.m_addr.size() == ADDR_TORV2_SIZE); + assert(ipIn.m_addr.size() == ADDR_TORV2_SIZE || ipIn.m_addr.size() == ADDR_TORV3_SIZE); + break; + case NET_I2P: + assert(ipIn.m_addr.size() == ADDR_I2P_SIZE); + break; + case NET_CJDNS: + assert(ipIn.m_addr.size() == ADDR_CJDNS_SIZE); break; case NET_INTERNAL: assert(ipIn.m_addr.size() == ADDR_INTERNAL_SIZE); @@ -150,24 +194,80 @@ bool CNetAddr::SetInternal(const std::string &name) return true; } +namespace torv3 { +// https://gitweb.torproject.org/torspec.git/tree/rend-spec-v3.txt#n2135 +static constexpr size_t CHECKSUM_LEN = 2; +static const unsigned char VERSION[] = {3}; +static constexpr size_t TOTAL_LEN = ADDR_TORV3_SIZE + CHECKSUM_LEN + sizeof(VERSION); + +static void Checksum(Span addr_pubkey, uint8_t (&checksum)[CHECKSUM_LEN]) +{ + // TORv3 CHECKSUM = H(".onion checksum" | PUBKEY | VERSION)[:2] + static const unsigned char prefix[] = ".onion checksum"; + static constexpr size_t prefix_len = 15; + + SHA3_256 hasher; + + hasher.Write(MakeSpan(prefix).first(prefix_len)); + hasher.Write(addr_pubkey); + hasher.Write(VERSION); + + uint8_t checksum_full[SHA3_256::OUTPUT_SIZE]; + + hasher.Finalize(checksum_full); + + memcpy(checksum, checksum_full, sizeof(checksum)); +} + +}; // namespace torv3 + /** - * Parse a TORv2 address and set this object to it. + * Parse a TOR address and set this object to it. * * @returns Whether or not the operation was successful. * * @see CNetAddr::IsTor() */ -bool CNetAddr::SetSpecial(const std::string& strName) +bool CNetAddr::SetSpecial(const std::string& str) { - if (strName.size()>6 && strName.substr(strName.size() - 6, 6) == ".onion") { - std::vector vchAddr = DecodeBase32(strName.substr(0, strName.size() - 6).c_str()); - if (vchAddr.size() != ADDR_TORV2_SIZE) { + static const char* suffix{".onion"}; + static constexpr size_t suffix_len{6}; + + if (!ValidAsCString(str) || str.size() <= suffix_len || + str.substr(str.size() - suffix_len) != suffix) { + return false; + } + + bool invalid; + const auto& input = DecodeBase32(str.substr(0, str.size() - suffix_len).c_str(), &invalid); + + if (invalid) { + return false; + } + + switch (input.size()) { + case ADDR_TORV2_SIZE: + m_net = NET_ONION; + m_addr.assign(input.begin(), input.end()); + return true; + case torv3::TOTAL_LEN: { + Span input_pubkey{input.data(), ADDR_TORV3_SIZE}; + Span input_checksum{input.data() + ADDR_TORV3_SIZE, torv3::CHECKSUM_LEN}; + Span input_version{input.data() + ADDR_TORV3_SIZE + torv3::CHECKSUM_LEN, sizeof(torv3::VERSION)}; + + uint8_t calculated_checksum[torv3::CHECKSUM_LEN]; + torv3::Checksum(input_pubkey, calculated_checksum); + + if (input_checksum != calculated_checksum || input_version != torv3::VERSION) { return false; } + m_net = NET_ONION; - m_addr.assign(vchAddr.begin(), vchAddr.end()); + m_addr.assign(input_pubkey.begin(), input_pubkey.end()); return true; } + } + return false; } @@ -284,13 +384,21 @@ bool CNetAddr::IsHeNet() const } /** - * @returns Whether or not this is a dummy address that maps an onion address - * into IPv6. - * + * Check whether this object represents a TOR address. * @see CNetAddr::SetSpecial(const std::string &) */ bool CNetAddr::IsTor() const { return m_net == NET_ONION; } +/** + * Check whether this object represents an I2P address. + */ +bool CNetAddr::IsI2P() const { return m_net == NET_I2P; } + +/** + * Check whether this object represents a CJDNS address. + */ +bool CNetAddr::IsCJDNS() const { return m_net == NET_CJDNS; } + bool CNetAddr::IsLocal() const { // IPv4 loopback (127.0.0.0/8 or 0.0.0.0/8) @@ -369,28 +477,72 @@ enum Network CNetAddr::GetNetwork() const return m_net; } +static std::string IPv6ToString(Span a) +{ + assert(a.size() == ADDR_IPV6_SIZE); + // clang-format off + return strprintf("%x:%x:%x:%x:%x:%x:%x:%x", + ReadBE16(&a[0]), + ReadBE16(&a[2]), + ReadBE16(&a[4]), + ReadBE16(&a[6]), + ReadBE16(&a[8]), + ReadBE16(&a[10]), + ReadBE16(&a[12]), + ReadBE16(&a[14])); + // clang-format on +} + std::string CNetAddr::ToStringIP() const { - if (IsTor()) - return EncodeBase32(m_addr) + ".onion"; - if (IsInternal()) + switch (m_net) { + case NET_IPV4: + case NET_IPV6: { + CService serv(*this, 0); + struct sockaddr_storage sockaddr; + socklen_t socklen = sizeof(sockaddr); + if (serv.GetSockAddr((struct sockaddr*)&sockaddr, &socklen)) { + char name[1025] = ""; + if (!getnameinfo((const struct sockaddr*)&sockaddr, socklen, name, + sizeof(name), nullptr, 0, NI_NUMERICHOST)) + return std::string(name); + } + if (m_net == NET_IPV4) { + return strprintf("%u.%u.%u.%u", m_addr[0], m_addr[1], m_addr[2], m_addr[3]); + } + return IPv6ToString(m_addr); + } + case NET_ONION: + switch (m_addr.size()) { + case ADDR_TORV2_SIZE: + return EncodeBase32(m_addr) + ".onion"; + case ADDR_TORV3_SIZE: { + + uint8_t checksum[torv3::CHECKSUM_LEN]; + torv3::Checksum(m_addr, checksum); + + // TORv3 onion_address = base32(PUBKEY | CHECKSUM | VERSION) + ".onion" + prevector address{m_addr.begin(), m_addr.end()}; + address.insert(address.end(), checksum, checksum + torv3::CHECKSUM_LEN); + address.insert(address.end(), torv3::VERSION, torv3::VERSION + sizeof(torv3::VERSION)); + + return EncodeBase32(address) + ".onion"; + } + default: + assert(false); + } + case NET_I2P: + return EncodeBase32(m_addr, false /* don't pad with = */) + ".b32.i2p"; + case NET_CJDNS: + return IPv6ToString(m_addr); + case NET_INTERNAL: return EncodeBase32(m_addr) + ".internal"; - CService serv(*this, 0); - struct sockaddr_storage sockaddr; - socklen_t socklen = sizeof(sockaddr); - if (serv.GetSockAddr((struct sockaddr*)&sockaddr, &socklen)) { - char name[1025] = ""; - if (!getnameinfo((const struct sockaddr*)&sockaddr, socklen, name, sizeof(name), nullptr, 0, NI_NUMERICHOST)) - return std::string(name); - } - if (IsIPv4()) - return strprintf("%u.%u.%u.%u", m_addr[0], m_addr[1], m_addr[2], m_addr[3]); - assert(IsIPv6()); - return strprintf("%x:%x:%x:%x:%x:%x:%x:%x", - m_addr[0] << 8 | m_addr[1], m_addr[2] << 8 | m_addr[3], - m_addr[4] << 8 | m_addr[5], m_addr[6] << 8 | m_addr[7], - m_addr[8] << 8 | m_addr[9], m_addr[10] << 8 | m_addr[11], - m_addr[12] << 8 | m_addr[13], m_addr[14] << 8 | m_addr[15]); + case NET_UNROUTABLE: // m_net is never and should not be set to NET_UNROUTABLE + case NET_MAX: // m_net is never and should not be set to NET_MAX + assert(false); + } // no default case, so the compiler can warn about missing cases + + assert(false); } std::string CNetAddr::ToString() const @@ -464,21 +616,22 @@ uint32_t CNetAddr::GetLinkedIPv4() const assert(false); } -uint32_t CNetAddr::GetNetClass() const { - uint32_t net_class = NET_IPV6; - if (IsLocal()) { - net_class = 255; - } +uint32_t CNetAddr::GetNetClass() const +{ + // Make sure that if we return NET_IPV6, then IsIPv6() is true. The callers expect that. + + // Check for "internal" first because such addresses are also !IsRoutable() + // and we don't want to return NET_UNROUTABLE in that case. if (IsInternal()) { - net_class = NET_INTERNAL; - } else if (!IsRoutable()) { - net_class = NET_UNROUTABLE; - } else if (HasLinkedIPv4()) { - net_class = NET_IPV4; - } else if (IsTor()) { - net_class = NET_ONION; + return NET_INTERNAL; } - return net_class; + if (!IsRoutable()) { + return NET_UNROUTABLE; + } + if (HasLinkedIPv4()) { + return NET_IPV4; + } + return m_net; } uint32_t CNetAddr::GetMappedAS(const std::vector &asmap) const { @@ -553,7 +706,7 @@ std::vector CNetAddr::GetGroup(const std::vector &asmap) co vchRet.push_back((ipv4 >> 24) & 0xFF); vchRet.push_back((ipv4 >> 16) & 0xFF); return vchRet; - } else if (IsTor()) { + } else if (IsTor() || IsI2P() || IsCJDNS()) { nBits = 4; } else if (IsHeNet()) { // for he.net, use /36 groups @@ -768,7 +921,7 @@ std::string CService::ToStringPort() const std::string CService::ToStringIPPort() const { - if (IsIPv4() || IsTor() || IsInternal()) { + if (IsIPv4() || IsTor() || IsI2P() || IsInternal()) { return ToStringIP() + ":" + ToStringPort(); } else { return "[" + ToStringIP() + "]:" + ToStringPort(); diff --git a/src/netaddress.h b/src/netaddress.h index 19d8f1f4435a4..5bc26bce0d6c3 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -51,9 +51,15 @@ enum Network /// IPv6 NET_IPV6, - /// TORv2 + /// TOR (v2 or v3) NET_ONION, + /// I2P + NET_I2P, + + /// CJDNS + NET_CJDNS, + /// A set of addresses that represent the hash of a string or FQDN. We use /// them in CAddrMan to keep track of which DNS seeds were used. NET_INTERNAL, @@ -94,6 +100,16 @@ static constexpr size_t ADDR_IPV6_SIZE = 16; /// Size of TORv2 address (in bytes). static constexpr size_t ADDR_TORV2_SIZE = 10; +/// Size of TORv3 address (in bytes). This is the length of just the address +/// as used in BIP155, without the checksum and the version byte. +static constexpr size_t ADDR_TORV3_SIZE = 32; + +/// Size of I2P address (in bytes). +static constexpr size_t ADDR_I2P_SIZE = 32; + +/// Size of CJDNS address (in bytes). +static constexpr size_t ADDR_CJDNS_SIZE = 16; + /// Size of "internal" (NET_INTERNAL) address (in bytes). static constexpr size_t ADDR_INTERNAL_SIZE = 10; @@ -155,6 +171,8 @@ class CNetAddr bool IsRFC6145() const; // IPv6 IPv4-translated address (::FFFF:0:0:0/96) bool IsHeNet() const; // IPv6 Hurricane Electric - https://he.net (2001:0470::/36) bool IsTor() const; + bool IsI2P() const; + bool IsCJDNS() const; bool IsLocal() const; bool IsRoutable() const; bool IsInternal() const; @@ -223,6 +241,9 @@ friend class CSubNet; IPV4 = 1, IPV6 = 2, TORV2 = 3, + TORV3 = 4, + I2P = 5, + CJDNS = 6, }; /** @@ -249,13 +270,12 @@ friend class CSubNet; * @retval true the network was recognized, is valid and `m_net` was set * @retval false not recognised (from future?) and should be silently ignored * @throws std::ios_base::failure if the network is one of the BIP155 founding - * networks recognized by this software (id 1..3) and has wrong address size. + * networks (id 1..6) with wrong address size. */ bool SetNetFromBIP155Network(uint8_t possible_bip155_net, size_t address_size); /** * Serialize in pre-ADDRv2/BIP155 format to an array. - * Some addresses (e.g. TORv3) cannot be serialized in pre-BIP155 format. */ void SerializeV1Array(uint8_t (&arr)[V1_SERIALIZATION_SIZE]) const { @@ -273,6 +293,9 @@ friend class CSubNet; memcpy(arr + prefix_size, m_addr.data(), m_addr.size()); return; case NET_ONION: + if (m_addr.size() == ADDR_TORV3_SIZE) { + break; + } prefix_size = sizeof(TORV2_IN_IPV6_PREFIX); assert(prefix_size + m_addr.size() == sizeof(arr)); memcpy(arr, TORV2_IN_IPV6_PREFIX.data(), prefix_size); @@ -284,17 +307,21 @@ friend class CSubNet; memcpy(arr, INTERNAL_IN_IPV6_PREFIX.data(), prefix_size); memcpy(arr + prefix_size, m_addr.data(), m_addr.size()); return; + case NET_I2P: + break; + case NET_CJDNS: + break; case NET_UNROUTABLE: case NET_MAX: assert(false); } // no default case, so the compiler can warn about missing cases - assert(false); + // Serialize TORv3, I2P and CJDNS as all-zeros. + memset(arr, 0x0, V1_SERIALIZATION_SIZE); } /** * Serialize in pre-ADDRv2/BIP155 format to a stream. - * Some addresses (e.g. TORv3) cannot be serialized in pre-BIP155 format. */ template void SerializeV1Stream(Stream& s) const diff --git a/src/test/base32_tests.cpp b/src/test/base32_tests.cpp index 941c519b6390d..f810632c24d22 100644 --- a/src/test/base32_tests.cpp +++ b/src/test/base32_tests.cpp @@ -13,10 +13,13 @@ BOOST_AUTO_TEST_CASE(base32_testvectors) { static const std::string vstrIn[] = {"","f","fo","foo","foob","fooba","foobar"}; static const std::string vstrOut[] = {"","my======","mzxq====","mzxw6===","mzxw6yq=","mzxw6ytb","mzxw6ytboi======"}; + static const std::string vstrOutNoPadding[] = {"","my","mzxq","mzxw6","mzxw6yq","mzxw6ytb","mzxw6ytboi"}; for (unsigned int i=0; i> addr; + BOOST_CHECK(addr.IsValid()); + BOOST_CHECK(addr.IsTor()); + BOOST_CHECK_EQUAL(addr.ToString(), + "pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion"); + BOOST_REQUIRE(s.empty()); + + // Invalid TORv3, with bogus length. + s << MakeSpan(ParseHex("04" // network type (TORv3) + "00" // address length + "00" // address + )); + BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure, + HasReason("BIP155 TORv3 address with length 0 (should be 32)")); + BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. + s.clear(); + + // Valid I2P. + s << MakeSpan(ParseHex("05" // network type (I2P) + "20" // address length + "a2894dabaec08c0051a481a6dac88b64" // address + "f98232ae42d4b6fd2fa81952dfe36a87")); + s >> addr; + BOOST_CHECK(addr.IsValid()); + BOOST_CHECK_EQUAL(addr.ToString(), + "ukeu3k5oycgaauneqgtnvselmt4yemvoilkln7jpvamvfx7dnkdq.b32.i2p"); + BOOST_REQUIRE(s.empty()); + + // Invalid I2P, with bogus length. + s << MakeSpan(ParseHex("05" // network type (I2P) + "03" // address length + "00" // address + )); + BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure, + HasReason("BIP155 I2P address with length 3 (should be 32)")); + BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. + s.clear(); + + // Valid CJDNS. + s << MakeSpan(ParseHex("06" // network type (CJDNS) + "10" // address length + "fc000001000200030004000500060007" // address + )); + s >> addr; + BOOST_CHECK(addr.IsValid()); + BOOST_CHECK_EQUAL(addr.ToString(), "fc00:1:2:3:4:5:6:7"); + BOOST_REQUIRE(s.empty()); + + // Invalid CJDNS, with bogus length. + s << MakeSpan(ParseHex("06" // network type (CJDNS) + "01" // address length + "00" // address + )); + BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure, + HasReason("BIP155 CJDNS address with length 1 (should be 16)")); + BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. + s.clear(); + // Unknown, with extreme length. s << MakeSpan(ParseHex("aa" // network type (unknown) "fe00000002" // address length (CompactSize's MAX_SIZE) diff --git a/src/utilstrencodings.cpp b/src/utilstrencodings.cpp index 7ce40d44a1ab5..f94154349f495 100644 --- a/src/utilstrencodings.cpp +++ b/src/utilstrencodings.cpp @@ -192,22 +192,27 @@ std::string DecodeBase64(const std::string& str) return std::string((const char*)vchRet.data(), vchRet.size()); } -std::string EncodeBase32(Span input) +std::string EncodeBase32(Span input, bool pad) { static const char *pbase32 = "abcdefghijklmnopqrstuvwxyz234567"; std::string str; str.reserve(((input.size() + 4) / 5) * 8); ConvertBits<8, 5, true>([&](int v) { str += pbase32[v]; }, input.begin(), input.end()); - while (str.size() % 8) str += '='; + if (pad) { + while (str.size() % 8) { + str += '='; + } + } return str; } -std::string EncodeBase32(const std::string& str) +std::string EncodeBase32(const std::string& str, bool pad) { - return EncodeBase32(MakeUCharSpan(str)); + return EncodeBase32(MakeUCharSpan(str), pad); } + std::vector DecodeBase32(const char* p, bool* pfInvalid) { static const int decode32_table[256] = diff --git a/src/utilstrencodings.h b/src/utilstrencodings.h index 022cfb5d8356e..d972a8ebb3ddb 100644 --- a/src/utilstrencodings.h +++ b/src/utilstrencodings.h @@ -59,8 +59,20 @@ std::string EncodeBase64(Span input); std::string EncodeBase64(const std::string& str); std::vector DecodeBase32(const char* p, bool* pfInvalid = NULL); std::string DecodeBase32(const std::string& str); -std::string EncodeBase32(Span input); -std::string EncodeBase32(const std::string& str); + +/** + * Base32 encode. + * If `pad` is true, then the output will be padded with '=' so that its length + * is a multiple of 8. + */ +std::string EncodeBase32(Span input, bool pad = true); + +/** + * Base32 encode. + * If `pad` is true, then the output will be padded with '=' so that its length + * is a multiple of 8. + */ +std::string EncodeBase32(const std::string& str, bool pad = true); std::string i64tostr(int64_t n); std::string itostr(int n); From 3eaa2733e76c2b4170357a14e32fcd9d45b627be Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Thu, 8 Oct 2020 20:27:27 -0700 Subject: [PATCH 31/58] Support bypassing range check in ReadCompactSize This is needed when we want to encode an arbitrary number as CompactSize like node service flags, which is a bitmask and could be bigger than the usual size of an object. --- src/serialize.h | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/serialize.h b/src/serialize.h index 9922532849dfb..b14ac89c8ff2a 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -33,7 +33,11 @@ class CScript; -static const unsigned int MAX_SIZE = 0x02000000; +/** + * The maximum size of a serialized object in bytes or number of elements + * (for eg vectors) when the size is encoded as CompactSize. + */ +static constexpr uint64_t MAX_SIZE = 0x02000000; /** Maximum amount of memory (in bytes) to allocate at once when deserializing vectors. */ static const unsigned int MAX_VECTOR_ALLOCATE = 5000000; @@ -334,8 +338,14 @@ void WriteCompactSize(Stream& os, uint64_t nSize) return; } -template -uint64_t ReadCompactSize(Stream& is) +/** + * Decode a CompactSize-encoded variable-length integer. + * + * As these are primarily used to encode the size of vector-like serializations, by default a range + * check is performed. When used as a generic number encoding, range_check should be set to false. + */ +template +uint64_t ReadCompactSize(Stream& is, bool range_check = true) { uint8_t chSize = ser_readdata8(is); uint64_t nSizeRet = 0; @@ -354,8 +364,9 @@ uint64_t ReadCompactSize(Stream& is) if (nSizeRet < 0x100000000ULL) throw std::ios_base::failure("non-canonical ReadCompactSize()"); } - if (nSizeRet > (uint64_t)MAX_SIZE) - throw std::ios_base::failure("ReadCompactSize() : size too large"); + if (range_check && nSizeRet > MAX_SIZE) { + throw std::ios_base::failure("ReadCompactSize(): size too large"); + } return nSizeRet; } @@ -489,7 +500,7 @@ static inline Wrapper Using(T&& t) { return Wrapper>(obj) #define VARINT(obj) Using>(obj) -#define COMPACTSIZE(obj) Using(obj) +#define COMPACTSIZE(obj) Using>(obj) #define LIMITED_STRING(obj,n) Using>(obj) /** Serialization wrapper class for integers in VarInt format. */ @@ -552,12 +563,13 @@ struct CustomUintFormatter template using BigEndianFormatter = CustomUintFormatter; /** Formatter for integers in CompactSize format. */ +template struct CompactSizeFormatter { template void Unser(Stream& s, I& v) { - uint64_t n = ReadCompactSize(s); + uint64_t n = ReadCompactSize(s, RangeCheck); if (n < std::numeric_limits::min() || n > std::numeric_limits::max()) { throw std::ios_base::failure("CompactSize exceeds limit of type"); } From b8c1dda66e6f184c5ed09f8619324720ea27c76e Mon Sep 17 00:00:00 2001 From: furszy Date: Thu, 10 Jun 2021 19:47:58 -0300 Subject: [PATCH 32/58] streams update: get rid of nType and nVersion. Plus support deserializing into temporaries --- src/streams.h | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/streams.h b/src/streams.h index 129a6c9a398ab..7fa9a2e88ab94 100644 --- a/src/streams.h +++ b/src/streams.h @@ -38,7 +38,7 @@ class OverrideStream OverrideStream& operator<<(const T& obj) { // Serialize to this stream - ::Serialize(*this->stream, obj, nType, nVersion); + ::Serialize(*this, obj); return (*this); } @@ -46,9 +46,23 @@ class OverrideStream OverrideStream& operator>>(T&& obj) { // Unserialize from this stream - ::Unserialize(*this->stream, obj, nType, nVersion); + ::Unserialize(*this, obj); return (*this); } + + void write(const char* pch, size_t nSize) + { + stream->write(pch, nSize); + } + + void read(char* pch, size_t nSize) + { + stream->read(pch, nSize); + } + + int GetVersion() const { return nVersion; } + int GetType() const { return nType; } + size_t size() const { return stream->size(); } }; /** Double ended buffer combining vector and stream-like interfaces. @@ -594,6 +608,9 @@ class CBufferedFile fclose(); } + int GetVersion() const { return nVersion; } + int GetType() const { return nType; } + // Disallow copies CBufferedFile(const CBufferedFile&) = delete; CBufferedFile& operator=(const CBufferedFile&) = delete; From cec9567408373324e381fc34c72fdc7f6759a0cc Mon Sep 17 00:00:00 2001 From: furszy Date: Fri, 11 Jun 2021 10:14:58 -0300 Subject: [PATCH 33/58] net: CAddress & CAddrMan: (un)serialize as ADDRv2 Change the serialization of `CAddrMan` to serialize its addresses in ADDRv2/BIP155 format by default. Introduce a new `CAddrMan` format version (3). Add support for ADDRv2 format in `CAddress` (un)serialization. Adaptation of btc@201a4596d92d640d5eb7e76cc8d959228fa09dbb --- doc/release-notes.md | 7 +++ src/addrman.h | 56 ++++++++++++++----- src/protocol.h | 12 ++++- src/streams.h | 1 + src/test/netbase_tests.cpp | 108 ++++++++++++++++++++++++++++++++++++- 5 files changed, 169 insertions(+), 15 deletions(-) diff --git a/doc/release-notes.md b/doc/release-notes.md index 20f9357d0b223..35bd4c2d116db 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -38,6 +38,13 @@ From PIVX Core 6.0 onwards, macOS versions earlier than 10.12 are no longer supp PIVX Core should also work on most other Unix-like systems but is not frequently tested on them. +The node's known peers are persisted to disk in a file called `peers.dat`. The +format of this file has been changed in a backwards-incompatible way in order to +accommodate the storage of Tor v3 and other BIP155 addresses. This means that if +the file is modified by v5.3 or newer then older versions will not be able to +read it. Those old versions, in the event of a downgrade, will log an error +message that deserialization has failed and will continue normal operation +as if the file was missing, creating a new empty one. (#2411) Notable Changes ============== diff --git a/src/addrman.h b/src/addrman.h index 96d340e826d8f..f272e9b23714d 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -13,6 +13,7 @@ #include "random.h" #include "sync.h" #include "timedata.h" +#include "tinyformat.h" #include "util/system.h" #include @@ -271,6 +272,14 @@ friend class CAddrManTest; void SetServices_(const CService& addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs); public: + //! Serialization versions. + enum class Format : uint8_t { + V0_HISTORICAL = 0, //!< historic format, before commit e6b343d88 + V1_DETERMINISTIC = 1, //!< for pre-asmap files + V2_ASMAP = 2, //!< for files including asmap version + V3_BIP155 = 3, //!< same as V2_ASMAP plus addresses are in BIP155 format + }; + // Compressed IP->ASN mapping, loaded from a file when a node starts. // Should be always empty if no file was provided. // This mapping is then used for bucketing nodes in Addrman. @@ -292,8 +301,8 @@ friend class CAddrManTest; /** - * serialized format: - * * version byte (1 for pre-asmap files, 2 for files including asmap version) + * Serialized format. + * * version byte (@see `Format`) * * 0x20 + nKey (serialized as if it were a vector, for backward compatibility) * * nNew * * nTried @@ -321,12 +330,15 @@ friend class CAddrManTest; * very little in common. */ template - void Serialize(Stream& s) const + void Serialize(Stream& s_) const { LOCK(cs); - unsigned char nVersion = 2; - s << nVersion; + // Always serialize in the latest version (currently Format::V3_BIP155). + + OverrideStream s(&s_, s_.GetType(), s_.GetVersion() | ADDRV2_FORMAT); + + s << static_cast(Format::V3_BIP155); s << ((unsigned char)32); s << nKey; s << nNew; @@ -378,13 +390,33 @@ friend class CAddrManTest; } template - void Unserialize(Stream& s) + void Unserialize(Stream& s_) { LOCK(cs); Clear(); - unsigned char nVersion; - s >> nVersion; + + Format format; + s_ >> Using>(format); + + static constexpr Format maximum_supported_format = Format::V3_BIP155; + if (format > maximum_supported_format) { + throw std::ios_base::failure(strprintf( + "Unsupported format of addrman database: %u. Maximum supported is %u. " + "Continuing operation without using the saved list of peers.", + static_cast(format), + static_cast(maximum_supported_format))); + } + + int stream_version = s_.GetVersion(); + if (format >= Format::V3_BIP155) { + // Add ADDRV2_FORMAT to the version so that the CNetAddr and CAddress + // unserialize methods know that an address in addrv2 format is coming. + stream_version |= ADDRV2_FORMAT; + } + + OverrideStream s(&s_, s_.GetType(), stream_version); + unsigned char nKeySize; s >> nKeySize; if (nKeySize != 32) throw std::ios_base::failure("Incorrect keysize in addrman deserialization"); @@ -393,7 +425,7 @@ friend class CAddrManTest; s >> nTried; int nUBuckets = 0; s >> nUBuckets; - if (nVersion != 0) { + if (format >= Format::V1_DETERMINISTIC) { nUBuckets ^= (1 << 30); } @@ -456,7 +488,7 @@ friend class CAddrManTest; supplied_asmap_version = SerializeHash(m_asmap); } uint256 serialized_asmap_version; - if (nVersion > 1) { + if (format >= Format::V2_ASMAP) { s >> serialized_asmap_version; } @@ -464,13 +496,13 @@ friend class CAddrManTest; CAddrInfo &info = mapInfo[n]; int bucket = entryToBucket[n]; int nUBucketPos = info.GetBucketPosition(nKey, true, bucket); - if (nVersion == 2 && nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && vvNew[bucket][nUBucketPos] == -1 && + if (format >= Format::V2_ASMAP && nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && vvNew[bucket][nUBucketPos] == -1 && info.nRefCount < ADDRMAN_NEW_BUCKETS_PER_ADDRESS && serialized_asmap_version == supplied_asmap_version) { // Bucketing has not changed, using existing bucket positions for the new table vvNew[bucket][nUBucketPos] = n; info.nRefCount++; } else { - // In case the new table data cannot be used (nVersion unknown, bucket count wrong or new asmap), + // In case the new table data cannot be used (format unknown, bucket count wrong or new asmap), // try to give them a reference based on their primary source address. LogPrint(BCLog::ADDRMAN, "Bucketing method was updated, re-bucketing addrman entries from disk\n"); bucket = info.GetNewBucket(nKey, m_asmap); diff --git a/src/protocol.h b/src/protocol.h index 3a49c1f69f658..500f04f923448 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -301,7 +301,8 @@ class CAddress : public CService public: CAddress() : CService{} {}; - explicit CAddress(CService ipIn, ServiceFlags nServicesIn) : CService{ipIn}, nServices{nServicesIn} {}; + CAddress(CService ipIn, ServiceFlags nServicesIn) : CService{ipIn}, nServices{nServicesIn} {}; + CAddress(CService ipIn, ServiceFlags nServicesIn, uint32_t nTimeIn) : CService{ipIn}, nServices{nServicesIn}, nTime{nTimeIn} {}; SERIALIZE_METHODS(CAddress, obj) { @@ -314,7 +315,14 @@ class CAddress : public CService (nVersion >= CADDR_TIME_VERSION && !(s.GetType() & SER_GETHASH))) { READWRITE(obj.nTime); } - READWRITE(Using>(obj.nServices)); + if (nVersion & ADDRV2_FORMAT) { + uint64_t services_tmp; + SER_WRITE(obj, services_tmp = obj.nServices); + READWRITE(Using>(services_tmp)); + SER_READ(obj, obj.nServices = static_cast(services_tmp)); + } else { + READWRITE(Using>(obj.nServices)); + } READWRITEAS(CService, obj); } diff --git a/src/streams.h b/src/streams.h index 7fa9a2e88ab94..3b9882397c796 100644 --- a/src/streams.h +++ b/src/streams.h @@ -63,6 +63,7 @@ class OverrideStream int GetVersion() const { return nVersion; } int GetType() const { return nType; } size_t size() const { return stream->size(); } + void ignore(size_t size) { return stream->ignore(size); } }; /** Double ended buffer combining vector and stream-like interfaces. diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 167beaf9df507..6f74b5ddd6b56 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -4,9 +4,14 @@ // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "test/test_pivx.h" + #include "net.h" // validateMasternodeIP #include "netbase.h" -#include "test/test_pivx.h" +#include "protocol.h" +#include "serialize.h" +#include "streams.h" +#include "version.h" #include @@ -385,4 +390,105 @@ BOOST_AUTO_TEST_CASE(netbase_dont_resolve_strings_with_embedded_nul_characters) BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0example.com\0", 35), ret)); } +// Since CNetAddr (un)ser is tested separately in net_tests.cpp here we only +// try a few edge cases for port, service flags and time. + +static const std::vector fixture_addresses({ + CAddress( + CService(CNetAddr(in6addr_loopback), 0 /* port */), + NODE_NONE, + 0x4966bc61U /* Fri Jan 9 02:54:25 UTC 2009 */ + ), + CAddress( + CService(CNetAddr(in6addr_loopback), 0x00f1 /* port */), + NODE_NETWORK, + 0x83766279U /* Tue Nov 22 11:22:33 UTC 2039 */ + ), + CAddress( + CService(CNetAddr(in6addr_loopback), 0xf1f2 /* port */), + static_cast(NODE_BLOOM), + 0xffffffffU /* Sun Feb 7 06:28:15 UTC 2106 */ + ) +}); + +// fixture_addresses should equal to this when serialized in V1 format. +// When this is unserialized from V1 format it should equal to fixture_addresses. +static constexpr const char* stream_addrv1_hex = + "03" // number of entries + + "61bc6649" // time, Fri Jan 9 02:54:25 UTC 2009 + "0000000000000000" // service flags, NODE_NONE + "00000000000000000000000000000001" // address, fixed 16 bytes (IPv4 embedded in IPv6) + "0000" // port + + "79627683" // time, Tue Nov 22 11:22:33 UTC 2039 + "0100000000000000" // service flags, NODE_NETWORK + "00000000000000000000000000000001" // address, fixed 16 bytes (IPv6) + "00f1" // port + + "ffffffff" // time, Sun Feb 7 06:28:15 UTC 2106 + "0400000000000000" // service flags, NODE_BLOOM + "00000000000000000000000000000001" // address, fixed 16 bytes (IPv6) + "f1f2"; // port + +// fixture_addresses should equal to this when serialized in V2 format. +// When this is unserialized from V2 format it should equal to fixture_addresses. +static constexpr const char* stream_addrv2_hex = + "03" // number of entries + + "61bc6649" // time, Fri Jan 9 02:54:25 UTC 2009 + "00" // service flags, COMPACTSIZE(NODE_NONE) + "02" // network id, IPv6 + "10" // address length, COMPACTSIZE(16) + "00000000000000000000000000000001" // address + "0000" // port + + "79627683" // time, Tue Nov 22 11:22:33 UTC 2039 + "01" // service flags, COMPACTSIZE(NODE_NETWORK) + "02" // network id, IPv6 + "10" // address length, COMPACTSIZE(16) + "00000000000000000000000000000001" // address + "00f1" // port + + "ffffffff" // time, Sun Feb 7 06:28:15 UTC 2106 + "04" // service flags, COMPACTSIZE(NODE_BLOOM) + "02" // network id, IPv6 + "10" // address length, COMPACTSIZE(16) + "00000000000000000000000000000001" // address + "f1f2"; // port + +BOOST_AUTO_TEST_CASE(caddress_serialize_v1) +{ + CDataStream s(SER_NETWORK, PROTOCOL_VERSION); + + s << fixture_addresses; + BOOST_CHECK_EQUAL(HexStr(s), stream_addrv1_hex); +} + +BOOST_AUTO_TEST_CASE(caddress_unserialize_v1) +{ + CDataStream s(ParseHex(stream_addrv1_hex), SER_NETWORK, PROTOCOL_VERSION); + std::vector addresses_unserialized; + + s >> addresses_unserialized; + BOOST_CHECK(fixture_addresses == addresses_unserialized); +} + +BOOST_AUTO_TEST_CASE(caddress_serialize_v2) +{ + CDataStream s(SER_NETWORK, PROTOCOL_VERSION | ADDRV2_FORMAT); + + s << fixture_addresses; + BOOST_CHECK_EQUAL(HexStr(s), stream_addrv2_hex); +} + +BOOST_AUTO_TEST_CASE(caddress_unserialize_v2) +{ + CDataStream s(ParseHex(stream_addrv2_hex), SER_NETWORK, PROTOCOL_VERSION | ADDRV2_FORMAT); + std::vector addresses_unserialized; + + s >> addresses_unserialized; + BOOST_CHECK(fixture_addresses == addresses_unserialized); +} + BOOST_AUTO_TEST_SUITE_END() From d496b64df30323d4f1f400e84a7aeabd0d464f6f Mon Sep 17 00:00:00 2001 From: Jonas Schnelli Date: Mon, 11 Dec 2017 09:54:13 -1000 Subject: [PATCH 34/58] [QA] fix mininode CAddress ser/deser --- test/functional/test_framework/messages.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) mode change 100644 => 100755 test/functional/test_framework/messages.py diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py old mode 100644 new mode 100755 index ad684d33b3b59..61fdae6d29575 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -192,19 +192,24 @@ def ToHex(obj): class CAddress(): def __init__(self): + self.time = 0 self.nServices = 1 self.pchReserved = b"\x00" * 10 + b"\xff" * 2 self.ip = "0.0.0.0" self.port = 0 - def deserialize(self, f): + def deserialize(self, f, with_time=True): + if with_time: + self.time = struct.unpack("H", f.read(2))[0] - def serialize(self): + def serialize(self, with_time=True): r = b"" + if with_time: + r += struct.pack("= 106: self.addrFrom = CAddress() - self.addrFrom.deserialize(f) + self.addrFrom.deserialize(f, False) self.nNonce = struct.unpack(" Date: Fri, 11 Jun 2021 12:01:45 -0300 Subject: [PATCH 35/58] Migrate to test_large_inv() to Misbehaving logging. --- test/functional/p2p_invalid_messages.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index 1de9619863f10..be1d7999cf490 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -85,12 +85,12 @@ def test_command(self): conn.sync_with_ping(timeout=1) self.nodes[0].disconnect_p2ps() - def test_large_inv(self): # future: add Misbehaving value check, first invalid message raise it to 20, second to 40. + def test_large_inv(self): conn = self.nodes[0].add_p2p_connection(P2PInterface()) - with self.nodes[0].assert_debug_log(['ERROR: peer=5 message inv size() = 50001']): + with self.nodes[0].assert_debug_log(['Misbehaving', 'peer=5 (0 -> 20): message inv size() = 50001']): msg = messages.msg_inv([messages.CInv(1, 1)] * 50001) conn.send_and_ping(msg) - with self.nodes[0].assert_debug_log(['ERROR: peer=5 message getdata size() = 50001']): + with self.nodes[0].assert_debug_log(['Misbehaving', 'peer=5 (20 -> 40): message getdata size() = 50001']): msg = messages.msg_getdata([messages.CInv(1, 1)] * 50001) conn.send_and_ping(msg) self.nodes[0].disconnect_p2ps() From 6da9a14ca4d978b3b523402bdc8f35afaa7dc730 Mon Sep 17 00:00:00 2001 From: furszy Date: Fri, 11 Jun 2021 12:09:20 -0300 Subject: [PATCH 36/58] net: advertise support for ADDRv2 via new message Introduce a new message `sendaddrv2` to signal support for ADDRv2. Send the new message immediately after sending the `VERACK` message. Add support for receiving and parsing ADDRv2 messages. Send ADDRv2 messages (instead of ADDR) to a peer if he has advertised support for it. Adaptation of btc@353a3fdaad055eea42a0baf7326bdd591f541170 with some further changes --- src/masternode-sync.cpp | 2 - src/net.h | 12 +- src/net_processing.cpp | 42 ++++- src/netaddress.cpp | 29 +++- src/netaddress.h | 6 + src/protocol.cpp | 4 + src/protocol.h | 12 ++ src/test/net_tests.cpp | 17 ++ test/functional/p2p_addrv2_relay.py | 78 +++++++++ test/functional/p2p_invalid_messages.py | 96 ++++++++++- test/functional/test_framework/messages.py | 185 +++++++++++++++------ test/functional/test_framework/mininode.py | 12 +- test/functional/test_runner.py | 1 + 13 files changed, 430 insertions(+), 66 deletions(-) create mode 100755 test/functional/p2p_addrv2_relay.py diff --git a/src/masternode-sync.cpp b/src/masternode-sync.cpp index 072542bfa98c2..b6aeda01a16f0 100644 --- a/src/masternode-sync.cpp +++ b/src/masternode-sync.cpp @@ -288,8 +288,6 @@ void CMasternodeSync::Process() return; } - LogPrint(BCLog::MASTERNODE, "CMasternodeSync::Process() - tick %d RequestedMasternodeAssets %d\n", tick, RequestedMasternodeAssets); - if (RequestedMasternodeAssets == MASTERNODE_SYNC_INITIAL) SwitchToNextAsset(); // sporks synced but blockchain is not, wait until we're almost at a recent block to continue diff --git a/src/net.h b/src/net.h index 4044f24e4db27..a8bafdd0eb0bd 100644 --- a/src/net.h +++ b/src/net.h @@ -571,6 +571,11 @@ class CNode bool fClient; const bool fInbound; bool fNetworkNode; + /** + * Whether the peer has signaled support for receiving ADDRv2 (BIP155) + * messages, implying a preference to receive ADDRv2 instead of ADDR ones. + */ + std::atomic_bool m_wants_addrv2{false}; std::atomic_bool fSuccessfullyConnected; std::atomic_bool fDisconnect; // We use fRelayTxes for two purposes - @@ -721,10 +726,15 @@ class CNode void PushAddress(const CAddress& _addr, FastRandomContext &insecure_rand) { + // Whether the peer supports the address in `_addr`. For example, + // nodes that do not implement BIP155 cannot receive Tor v3 addresses + // because they require ADDRv2 (BIP155) encoding. + const bool addr_format_supported = m_wants_addrv2 || _addr.IsAddrV1Compatible(); + // Known checking here is only to save space from duplicates. // SendMessages will filter it again for knowns that were added // after addresses were pushed. - if (_addr.IsValid() && !addrKnown.contains(_addr.GetKey())) { + if (_addr.IsValid() && !addrKnown.contains(_addr.GetKey()) && addr_format_supported) { if (vAddrToSend.size() >= MAX_ADDR_TO_SEND) { vAddrToSend[insecure_rand.randrange(vAddrToSend.size())] = _addr; } else { diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 73d2fad66684a..79643c69b9d69 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -18,6 +18,7 @@ #include "primitives/block.h" #include "primitives/transaction.h" #include "sporkdb.h" +#include "streams.h" int64_t nTimeBestReceived = 0; // Used only to inform the wallet of when we last received a block @@ -1146,8 +1147,6 @@ bool static ProcessMessage(CNode* pfrom, std::string strCommand, CDataStream& vR if (pfrom->fInbound) PushNodeVersion(pfrom, connman, GetAdjustedTime()); - connman->PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERACK)); - pfrom->nServices = nServices; pfrom->SetAddrLocal(addrMe); { @@ -1171,6 +1170,12 @@ bool static ProcessMessage(CNode* pfrom, std::string strCommand, CDataStream& vR pfrom->SetSendVersion(nSendVersion); pfrom->nVersion = nVersion; + CNetMsgMaker msg_maker(INIT_PROTO_VERSION); + connman->PushMessage(pfrom, msg_maker.Make(NetMsgType::VERACK)); + + // Signal ADDRv2 support (BIP155). + connman->PushMessage(pfrom, msg_maker.Make(NetMsgType::SENDADDRV2)); + { LOCK(cs_main); // Potentially mark this peer as a preferred download peer. @@ -1270,16 +1275,24 @@ bool static ProcessMessage(CNode* pfrom, std::string strCommand, CDataStream& vR } - else if (strCommand == NetMsgType::ADDR) { + else if (strCommand == NetMsgType::ADDR || strCommand == NetMsgType::ADDRV2) { + int stream_version = vRecv.GetVersion(); + if (strCommand == NetMsgType::ADDRV2) { + // Add ADDRV2_FORMAT to the version so that the CNetAddr and CAddress + // unserialize methods know that an address in v2 format is coming. + stream_version |= ADDRV2_FORMAT; + } + + OverrideStream s(&vRecv, vRecv.GetType(), stream_version); std::vector vAddr; - vRecv >> vAddr; + s >> vAddr; // Don't want addr from older versions unless seeding if (pfrom->nVersion < CADDR_TIME_VERSION && connman->GetAddressCount() > 1000) return true; if (vAddr.size() > MAX_ADDR_TO_SEND) { LOCK(cs_main); - Misbehaving(pfrom->GetId(), 20, strprintf("message addr size() = %u", vAddr.size())); + Misbehaving(pfrom->GetId(), 20, strprintf("%s message size = %u", strCommand, vAddr.size())); return false; } @@ -1313,6 +1326,10 @@ bool static ProcessMessage(CNode* pfrom, std::string strCommand, CDataStream& vR pfrom->fDisconnect = true; } + else if (strCommand == NetMsgType::SENDADDRV2) { + pfrom->m_wants_addrv2 = true; + return true; + } else if (strCommand == NetMsgType::INV) { std::vector vInv; @@ -2089,20 +2106,31 @@ bool PeerLogicValidation::SendMessages(CNode* pto, std::atomic& interruptM pto->nNextAddrSend = PoissonNextSend(nNow, AVG_ADDRESS_BROADCAST_INTERVAL); std::vector vAddr; vAddr.reserve(pto->vAddrToSend.size()); + + const char* msg_type; + int make_flags; + if (pto->m_wants_addrv2) { + msg_type = NetMsgType::ADDRV2; + make_flags = ADDRV2_FORMAT; + } else { + msg_type = NetMsgType::ADDR; + make_flags = 0; + } + for (const CAddress& addr : pto->vAddrToSend) { if (!pto->addrKnown.contains(addr.GetKey())) { pto->addrKnown.insert(addr.GetKey()); vAddr.push_back(addr); // receiver rejects addr messages larger than 1000 if (vAddr.size() >= 1000) { - connman->PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); + connman->PushMessage(pto, msgMaker.Make(make_flags, msg_type, vAddr)); vAddr.clear(); } } } pto->vAddrToSend.clear(); if (!vAddr.empty()) - connman->PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); + connman->PushMessage(pto, msgMaker.Make(make_flags, msg_type, vAddr)); } // Start block sync diff --git a/src/netaddress.cpp b/src/netaddress.cpp index dcb93e0c51dfd..a930f5d2318ff 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -466,6 +466,26 @@ bool CNetAddr::IsInternal() const return m_net == NET_INTERNAL; } +bool CNetAddr::IsAddrV1Compatible() const +{ + switch (m_net) { + case NET_IPV4: + case NET_IPV6: + case NET_INTERNAL: + return true; + case NET_ONION: + return m_addr.size() == ADDR_TORV2_SIZE; + case NET_I2P: + case NET_CJDNS: + return false; + case NET_UNROUTABLE: // m_net is never and should not be set to NET_UNROUTABLE + case NET_MAX: // m_net is never and should not be set to NET_MAX + assert(false); + } // no default case, so the compiler can warn about missing cases + + assert(false); +} + enum Network CNetAddr::GetNetwork() const { if (IsInternal()) @@ -731,9 +751,12 @@ std::vector CNetAddr::GetGroup(const std::vector &asmap) co std::vector CNetAddr::GetAddrBytes() const { - uint8_t serialized[V1_SERIALIZATION_SIZE]; - SerializeV1Array(serialized); - return {std::begin(serialized), std::end(serialized)}; + if (IsAddrV1Compatible()) { + uint8_t serialized[V1_SERIALIZATION_SIZE]; + SerializeV1Array(serialized); + return {std::begin(serialized), std::end(serialized)}; + } + return std::vector(m_addr.begin(), m_addr.end()); } uint64_t CNetAddr::GetHash() const diff --git a/src/netaddress.h b/src/netaddress.h index 5bc26bce0d6c3..7c0c19a5e5fbe 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -177,6 +177,12 @@ class CNetAddr bool IsRoutable() const; bool IsInternal() const; bool IsValid() const; + + /** + * Check if the current object can be serialized in pre-ADDRv2/BIP155 format. + */ + bool IsAddrV1Compatible() const; + enum Network GetNetwork() const; std::string ToString() const; std::string ToStringIP() const; diff --git a/src/protocol.cpp b/src/protocol.cpp index 81e353c849a60..6109c65097eff 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -19,6 +19,8 @@ namespace NetMsgType const char* VERSION = "version"; const char* VERACK = "verack"; const char* ADDR = "addr"; +const char* ADDRV2="addrv2"; +const char* SENDADDRV2="sendaddrv2"; const char* INV = "inv"; const char* GETDATA = "getdata"; const char* MERKLEBLOCK = "merkleblock"; @@ -59,6 +61,8 @@ const static std::string allNetMessageTypes[] = { NetMsgType::VERSION, NetMsgType::VERACK, NetMsgType::ADDR, + NetMsgType::ADDRV2, + NetMsgType::SENDADDRV2, NetMsgType::INV, NetMsgType::GETDATA, NetMsgType::MERKLEBLOCK, diff --git a/src/protocol.h b/src/protocol.h index 500f04f923448..363aa56b96654 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -83,6 +83,18 @@ extern const char* VERACK; * @see https://bitcoin.org/en/developer-reference#addr */ extern const char* ADDR; +/** + * The addrv2 message relays connection information for peers on the network just + * like the addr message, but is extended to allow gossiping of longer node + * addresses (see BIP155). + */ +extern const char *ADDRV2; +/** + * The sendaddrv2 message signals support for receiving ADDRV2 messages (BIP155). + * It also implies that its sender can encode as ADDRV2 and would send ADDRV2 + * instead of ADDR to a peer that has signaled ADDRV2 support by sending SENDADDRV2. + */ +extern const char *SENDADDRV2; /** * The inv message (inventory message) transmits one or more inventories of * objects known to the transmitting peer. diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index b6e5fd39a0930..f4edc432d93dd 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -203,6 +203,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) BOOST_REQUIRE(addr.IsIPv4()); BOOST_CHECK(addr.IsBindAny()); + BOOST_CHECK(addr.IsAddrV1Compatible()); BOOST_CHECK_EQUAL(addr.ToString(), "0.0.0.0"); // IPv4, INADDR_NONE @@ -211,6 +212,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) BOOST_REQUIRE(addr.IsIPv4()); BOOST_CHECK(!addr.IsBindAny()); + BOOST_CHECK(addr.IsAddrV1Compatible()); BOOST_CHECK_EQUAL(addr.ToString(), "255.255.255.255"); // IPv4, casual @@ -219,6 +221,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) BOOST_REQUIRE(addr.IsIPv4()); BOOST_CHECK(!addr.IsBindAny()); + BOOST_CHECK(addr.IsAddrV1Compatible()); BOOST_CHECK_EQUAL(addr.ToString(), "12.34.56.78"); // IPv6, in6addr_any @@ -227,6 +230,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) BOOST_REQUIRE(addr.IsIPv6()); BOOST_CHECK(addr.IsBindAny()); + BOOST_CHECK(addr.IsAddrV1Compatible()); BOOST_CHECK_EQUAL(addr.ToString(), "::"); // IPv6, casual @@ -235,6 +239,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) BOOST_REQUIRE(addr.IsIPv6()); BOOST_CHECK(!addr.IsBindAny()); + BOOST_CHECK(addr.IsAddrV1Compatible()); BOOST_CHECK_EQUAL(addr.ToString(), "1122:3344:5566:7788:9900:aabb:ccdd:eeff"); // TORv2 @@ -243,6 +248,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) BOOST_REQUIRE(addr.IsTor()); BOOST_CHECK(!addr.IsBindAny()); + BOOST_CHECK(addr.IsAddrV1Compatible()); BOOST_CHECK_EQUAL(addr.ToString(), "6hzph5hv6337r6p2.onion"); // TORv3 @@ -252,6 +258,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) BOOST_REQUIRE(addr.IsTor()); BOOST_CHECK(!addr.IsBindAny()); + BOOST_CHECK(!addr.IsAddrV1Compatible()); BOOST_CHECK_EQUAL(addr.ToString(), torv3_addr); // TORv3, broken, with wrong checksum @@ -276,6 +283,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) BOOST_REQUIRE(addr.IsInternal()); BOOST_CHECK(!addr.IsBindAny()); + BOOST_CHECK(addr.IsAddrV1Compatible()); BOOST_CHECK_EQUAL(addr.ToString(), "esffpvrt3wpeaygy.internal"); // Totally bogus @@ -370,6 +378,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) s >> addr; BOOST_CHECK(addr.IsValid()); BOOST_CHECK(addr.IsIPv4()); + BOOST_CHECK(addr.IsAddrV1Compatible()); BOOST_CHECK_EQUAL(addr.ToString(), "1.2.3.4"); BOOST_REQUIRE(s.empty()); @@ -406,6 +415,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) s >> addr; BOOST_CHECK(addr.IsValid()); BOOST_CHECK(addr.IsIPv6()); + BOOST_CHECK(addr.IsAddrV1Compatible()); BOOST_CHECK_EQUAL(addr.ToString(), "102:304:506:708:90a:b0c:d0e:f10"); BOOST_REQUIRE(s.empty()); @@ -417,6 +427,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) // sha256(name)[0:10] s >> addr; BOOST_CHECK(addr.IsInternal()); + BOOST_CHECK(addr.IsAddrV1Compatible()); BOOST_CHECK_EQUAL(addr.ToString(), "zklycewkdo64v6wc.internal"); BOOST_REQUIRE(s.empty()); @@ -452,6 +463,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) s >> addr; BOOST_CHECK(addr.IsValid()); BOOST_CHECK(addr.IsTor()); + BOOST_CHECK(addr.IsAddrV1Compatible()); BOOST_CHECK_EQUAL(addr.ToString(), "6hzph5hv6337r6p2.onion"); BOOST_REQUIRE(s.empty()); @@ -473,6 +485,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) s >> addr; BOOST_CHECK(addr.IsValid()); BOOST_CHECK(addr.IsTor()); + BOOST_CHECK(!addr.IsAddrV1Compatible()); BOOST_CHECK_EQUAL(addr.ToString(), "pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion"); BOOST_REQUIRE(s.empty()); @@ -494,6 +507,8 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) "f98232ae42d4b6fd2fa81952dfe36a87")); s >> addr; BOOST_CHECK(addr.IsValid()); + BOOST_CHECK(addr.IsI2P()); + BOOST_CHECK(!addr.IsAddrV1Compatible()); BOOST_CHECK_EQUAL(addr.ToString(), "ukeu3k5oycgaauneqgtnvselmt4yemvoilkln7jpvamvfx7dnkdq.b32.i2p"); BOOST_REQUIRE(s.empty()); @@ -515,6 +530,8 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) )); s >> addr; BOOST_CHECK(addr.IsValid()); + BOOST_CHECK(addr.IsCJDNS()); + BOOST_CHECK(!addr.IsAddrV1Compatible()); BOOST_CHECK_EQUAL(addr.ToString(), "fc00:1:2:3:4:5:6:7"); BOOST_REQUIRE(s.empty()); diff --git a/test/functional/p2p_addrv2_relay.py b/test/functional/p2p_addrv2_relay.py new file mode 100755 index 0000000000000..b6cb9b19f0030 --- /dev/null +++ b/test/functional/p2p_addrv2_relay.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test addrv2 relay +""" + +import time + +from test_framework.messages import ( + CAddress, + msg_addrv2, + NODE_NETWORK +) +from test_framework.mininode import P2PInterface +from test_framework.test_framework import PivxTestFramework +from test_framework.util import assert_equal + +ADDRS = [] +for i in range(10): + addr = CAddress() + addr.time = int(time.time()) + i + addr.nServices = NODE_NETWORK + addr.ip = "123.123.123.{}".format(i % 256) + addr.port = 8333 + i + ADDRS.append(addr) + + +class AddrReceiver(P2PInterface): + addrv2_received_and_checked = False + + def __init__(self): + super().__init__(support_addrv2 = True) + + def on_addrv2(self, message): + for addr in message.addrs: + assert_equal(addr.nServices, 1) # NODE_NETWORK + assert addr.ip.startswith('123.123.123.') + assert (8333 <= addr.port < 8343) + self.addrv2_received_and_checked = True + + def wait_for_addrv2(self): + self.wait_until(lambda: "addrv2" in self.last_message) + + +class AddrTest(PivxTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def run_test(self): + self.log.info('Create connection that sends addrv2 messages') + addr_source = self.nodes[0].add_p2p_connection(P2PInterface()) + msg = msg_addrv2() + + self.log.info('Send too-large addrv2 message') + msg.addrs = ADDRS * 101 + with self.nodes[0].assert_debug_log(['addrv2 message size = 1010']): + addr_source.send_and_ping(msg) + + self.log.info('Check that addrv2 message content is relayed and added to addrman') + addr_receiver = self.nodes[0].add_p2p_connection(AddrReceiver()) + msg.addrs = ADDRS + with self.nodes[0].assert_debug_log([ + 'Added 10 addresses from 127.0.0.1: 0 tried', + 'received: addrv2 (131 bytes) peer=1', + 'sending addrv2 (131 bytes) peer=2', + ]): + addr_source.send_and_ping(msg) + self.nodes[0].setmocktime(int(time.time()) + 30 * 60) + addr_receiver.wait_for_addrv2() + + assert addr_receiver.addrv2_received_and_checked + + +if __name__ == '__main__': + AddrTest().main() diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index be1d7999cf490..a5a29ba9e4b40 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -4,12 +4,18 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid network messages.""" +import struct +import time + from test_framework import messages from test_framework.mininode import ( P2PDataStore, P2PInterface, ) from test_framework.test_framework import PivxTestFramework +from test_framework.util import ( + hex_str_to_bytes, +) MSG_LIMIT = 2 * 1024 * 1024 # 2MB, per MAX_PROTOCOL_MESSAGE_LENGTH VALID_DATA_LIMIT = MSG_LIMIT - 5 # Account for the 5-byte length prefix @@ -29,6 +35,10 @@ def __repr__(self): return "{}(data={})".format(self.command, self.str_data) +class SenderOfAddrV2(P2PInterface): + def wait_for_sendaddrv2(self): + self.wait_until(lambda: 'sendaddrv2' in self.last_message) + class InvalidMessagesTest(PivxTestFramework): def set_test_params(self): self.num_nodes = 1 @@ -39,6 +49,10 @@ def run_test(self): self.test_checksum() self.test_size() self.test_command() + self.test_addrv2_empty() + self.test_addrv2_no_addresses() + self.test_addrv2_too_long_address() + self.test_addrv2_unrecognized_network() self.test_large_inv() self.test_resource_exhaustion() @@ -85,12 +99,90 @@ def test_command(self): conn.sync_with_ping(timeout=1) self.nodes[0].disconnect_p2ps() + def test_addrv2(self, label, required_log_messages, raw_addrv2): + node = self.nodes[0] + conn = node.add_p2p_connection(SenderOfAddrV2()) + + # Make sure pivxd signals support for ADDRv2, otherwise this test + # will bombard an old node with messages it does not recognize which + # will produce unexpected results. + conn.wait_for_sendaddrv2() + + self.log.info('Test addrv2: ' + label) + + msg = msg_unrecognized(str_data=b'') + msg.command = b'addrv2' + with node.assert_debug_log(required_log_messages): + # override serialize() which would include the length of the data + msg.serialize = lambda: raw_addrv2 + conn.send_raw_message(conn.build_message(msg)) + conn.sync_with_ping() + + node.disconnect_p2ps() + + def test_addrv2_empty(self): + self.test_addrv2('empty', + [ + 'received: addrv2 (0 bytes)', + 'ProcessMessages(addrv2, 0 bytes): Exception', + 'end of data', + ], + b'') + + def test_addrv2_no_addresses(self): + self.test_addrv2('no addresses', + [ + 'received: addrv2 (1 bytes)', + ], + hex_str_to_bytes('00')) + + def test_addrv2_too_long_address(self): + self.test_addrv2('too long address', + [ + 'received: addrv2 (525 bytes)', + 'ProcessMessage(addrv2, 525 bytes) FAILED', + 'Address too long: 513 > 512', + ], + hex_str_to_bytes( + '01' + # number of entries + '61bc6649' + # time, Fri Jan 9 02:54:25 UTC 2009 + '00' + # service flags, COMPACTSIZE(NODE_NONE) + '01' + # network type (IPv4) + 'fd0102' + # address length (COMPACTSIZE(513)) + 'ab' * 513 + # address + '208d')) # port + + def test_addrv2_unrecognized_network(self): + now_hex = struct.pack(' 20): message inv size() = 50001']): + with self.nodes[0].assert_debug_log(['Misbehaving', 'peer=9 (0 -> 20): message inv size() = 50001']): msg = messages.msg_inv([messages.CInv(1, 1)] * 50001) conn.send_and_ping(msg) - with self.nodes[0].assert_debug_log(['Misbehaving', 'peer=5 (20 -> 40): message getdata size() = 50001']): + with self.nodes[0].assert_debug_log(['Misbehaving', 'peer=9 (20 -> 40): message getdata size() = 50001']): msg = messages.msg_getdata([messages.CInv(1, 1)] * 50001) conn.send_and_ping(msg) self.nodes[0].disconnect_p2ps() diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 61fdae6d29575..770c058cde18c 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -124,12 +124,17 @@ def uint256_from_compact(c): return v -def deser_vector(f, c): +# deser_function_name: Allow for an alternate deserialization function on the +# entries in the vector. +def deser_vector(f, c, deser_function_name=None): nit = deser_compact_size(f) r = [] - for i in range(nit): + for _ in range(nit): t = c() - t.deserialize(f) + if deser_function_name: + getattr(t, deser_function_name)(f) + else: + t.deserialize(f) r.append(t) return r @@ -190,37 +195,85 @@ def ToHex(obj): # Objects that map to pivxd objects, which can be serialized/deserialized -class CAddress(): +class CAddress: + __slots__ = ("net", "ip", "nServices", "port", "time") + + # see https://github.com/bitcoin/bips/blob/master/bip-0155.mediawiki + NET_IPV4 = 1 + + ADDRV2_NET_NAME = { + NET_IPV4: "IPv4" + } + + ADDRV2_ADDRESS_LENGTH = { + NET_IPV4: 4 + } + def __init__(self): self.time = 0 self.nServices = 1 - self.pchReserved = b"\x00" * 10 + b"\xff" * 2 + self.net = self.NET_IPV4 self.ip = "0.0.0.0" self.port = 0 - def deserialize(self, f, with_time=True): + def deserialize(self, f, *, with_time=True): + """Deserialize from addrv1 format (pre-BIP155)""" if with_time: - self.time = struct.unpack("H", f.read(2))[0] - def serialize(self, with_time=True): + def serialize(self, *, with_time=True): + """Serialize in addrv1 format (pre-BIP155)""" + assert self.net == self.NET_IPV4 r = b"" if with_time: - r += struct.pack("H", self.port) + return r + + def deserialize_v2(self, f): + """Deserialize from addrv2 format (BIP155)""" + self.time = struct.unpack("H", f.read(2))[0] + + def serialize_v2(self): + """Serialize in addrv2 format (BIP155)""" + assert self.net == self.NET_IPV4 + r = b"" + r += struct.pack("H", self.port) return r def __repr__(self): - return "CAddress(nServices=%i ip=%s port=%i)" % (self.nServices, - self.ip, self.port) + return ("CAddress(nServices=%i net=%s addr=%s port=%i)" + % (self.nServices, self.ADDRV2_NET_NAME[self.net], self.ip, self.port)) -class CInv(): +class CInv: typemap = { 0: "MSG_ERROR", 1: "MSG_TX", @@ -264,7 +317,7 @@ def __repr__(self): % (self.typemap[self.type], self.hash) -class CBlockLocator(): +class CBlockLocator: def __init__(self): self.nVersion = MY_VERSION self.vHave = [] @@ -284,7 +337,7 @@ def __repr__(self): % (self.nVersion, repr(self.vHave)) -class COutPoint(): +class COutPoint: def __init__(self, hash=0, n=0): self.hash = hash self.n = n @@ -317,7 +370,7 @@ def to_json(self): NullOutPoint = COutPoint(0, 0xffffffff) -class CTxIn(): +class CTxIn: def __init__(self, outpoint=None, scriptSig=b"", nSequence=0): if outpoint is None: self.prevout = COutPoint() @@ -348,7 +401,7 @@ def is_zerocoinspend(self): return bytes_to_hex_str(self.scriptSig)[:2] == "c2" -class CTxOut(): +class CTxOut: def __init__(self, nValue=0, scriptPubKey=b""): self.nValue = nValue self.scriptPubKey = scriptPubKey @@ -369,7 +422,7 @@ def __repr__(self): bytes_to_hex_str(self.scriptPubKey)) -class CTransaction(): +class CTransaction: def __init__(self, tx=None): if tx is None: self.nVersion = 1 @@ -459,7 +512,7 @@ def __repr__(self): % (self.nVersion, repr(self.vin), repr(self.vout), self.nLockTime) -class CBlockHeader(): +class CBlockHeader: def __init__(self, header=None): if header is None: self.set_null() @@ -655,7 +708,7 @@ def __repr__(self): time.ctime(self.nTime), self.nBits, self.nNonce, repr(self.vtx)) -class PrefilledTransaction(): +class PrefilledTransaction: def __init__(self, index=0, tx = None): self.index = index self.tx = tx @@ -684,7 +737,7 @@ def __repr__(self): return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx)) # This is what we send on the wire, in a cmpctblock message. -class P2PHeaderAndShortIDs(): +class P2PHeaderAndShortIDs: def __init__(self): self.header = CBlockHeader() self.nonce = 0 @@ -736,7 +789,7 @@ def calculate_shortid(k0, k1, tx_hash): # This version gets rid of the array lengths, and reinterprets the differential # encoding into indices that can be used for lookup. -class HeaderAndShortIDs(): +class HeaderAndShortIDs: def __init__(self, p2pheaders_and_shortids = None): self.header = CBlockHeader() self.nonce = 0 @@ -797,7 +850,7 @@ def __repr__(self): return "HeaderAndShortIDs(header=%s, nonce=%d, shortids=%s, prefilledtxn=%s" % (repr(self.header), self.nonce, repr(self.shortids), repr(self.prefilled_txn)) -class BlockTransactionsRequest(): +class BlockTransactionsRequest: def __init__(self, blockhash=0, indexes = None): self.blockhash = blockhash @@ -837,7 +890,7 @@ def __repr__(self): return "BlockTransactionsRequest(hash=%064x indexes=%s)" % (self.blockhash, repr(self.indexes)) -class BlockTransactions(): +class BlockTransactions: def __init__(self, blockhash=0, transactions = None): self.blockhash = blockhash @@ -859,7 +912,7 @@ def serialize(self, with_witness=True): def __repr__(self): return "BlockTransactions(hash=%064x transactions=%s)" % (self.blockhash, repr(self.transactions)) -class CPartialMerkleTree(): +class CPartialMerkleTree: def __init__(self): self.nTransactions = 0 self.vHash = [] @@ -887,7 +940,7 @@ def serialize(self): def __repr__(self): return "CPartialMerkleTree(nTransactions=%d, vHash=%s, vBits=%s)" % (self.nTransactions, repr(self.vHash), repr(self.vBits)) -class CMerkleBlock(): +class CMerkleBlock: def __init__(self): self.header = CBlockHeader() self.txn = CPartialMerkleTree() @@ -907,7 +960,7 @@ def __repr__(self): # Objects that correspond to messages on the wire -class msg_version(): +class msg_version: command = b"version" def __init__(self): @@ -928,11 +981,11 @@ def deserialize(self, f): self.nServices = struct.unpack("= 106: self.addrFrom = CAddress() - self.addrFrom.deserialize(f, False) + self.addrFrom.deserialize(f, with_time=False) self.nNonce = struct.unpack(" -class msg_headers(): +class msg_headers: command = b"headers" def __init__(self, headers=None): @@ -1254,7 +1339,7 @@ def serialize(self): def __repr__(self): return "msg_headers(headers=%s)" % repr(self.headers) -class msg_feefilter(): +class msg_feefilter: command = b"feefilter" def __init__(self, feerate=0): @@ -1271,7 +1356,7 @@ def serialize(self): def __repr__(self): return "msg_feefilter(feerate=%08x)" % self.feerate -class msg_sendcmpct(): +class msg_sendcmpct: command = b"sendcmpct" def __init__(self): @@ -1291,7 +1376,7 @@ def serialize(self): def __repr__(self): return "msg_sendcmpct(announce=%s, version=%lu)" % (self.announce, self.version) -class msg_cmpctblock(): +class msg_cmpctblock: command = b"cmpctblock" def __init__(self, header_and_shortids = None): @@ -1309,7 +1394,7 @@ def serialize(self): def __repr__(self): return "msg_cmpctblock(HeaderAndShortIDs=%s)" % repr(self.header_and_shortids) -class msg_getblocktxn(): +class msg_getblocktxn: command = b"getblocktxn" def __init__(self): diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index 5b6feecb1088f..9f8540a205cd0 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -25,6 +25,7 @@ from .messages import ( msg_addr, + msg_addrv2, msg_block, msg_blocktxn, msg_cmpctblock, @@ -39,6 +40,7 @@ msg_mempool, msg_ping, msg_pong, + msg_sendaddrv2, msg_sendcmpct, msg_sendheaders, msg_tx, @@ -57,6 +59,7 @@ MESSAGEMAP = { b"addr": msg_addr, + b"addrv2": msg_addrv2, b"block": msg_block, b"blocktxn": msg_blocktxn, b"cmpctblock": msg_cmpctblock, @@ -71,6 +74,7 @@ b"mempool": msg_mempool, b"ping": msg_ping, b"pong": msg_pong, + b"sendaddrv2": msg_sendaddrv2, b"sendcmpct": msg_sendcmpct, b"sendheaders": msg_sendheaders, b"tx": msg_tx, @@ -260,7 +264,7 @@ class P2PInterface(P2PConnection): Individual testcases should subclass this and override the on_* methods if they want to alter message handling behaviour.""" - def __init__(self): + def __init__(self, support_addrv2=False): super().__init__() # Track number of messages of each type received and the most recent @@ -274,6 +278,8 @@ def __init__(self): # The network services received from the peer self.nServices = 0 + self.support_addrv2 = support_addrv2 + def peer_connect(self, *args, services=NODE_NETWORK, send_version=True, **kwargs): create_conn = super().peer_connect(*args, **kwargs) @@ -316,6 +322,7 @@ def on_close(self): pass def on_addr(self, message): pass + def on_addrv2(self, message): pass def on_block(self, message): pass def on_blocktxn(self, message): pass def on_cmpctblock(self, message): pass @@ -328,6 +335,7 @@ def on_getheaders(self, message): pass def on_headers(self, message): pass def on_mempool(self, message): pass def on_pong(self, message): pass + def on_sendaddrv2(self, message): pass def on_sendcmpct(self, message): pass def on_sendheaders(self, message): pass def on_tx(self, message): pass @@ -349,6 +357,8 @@ def on_verack(self, message): def on_version(self, message): assert message.nVersion >= MIN_VERSION_SUPPORTED, "Version {} received. Test framework only supports versions greater than {}".format(message.nVersion, MIN_VERSION_SUPPORTED) self.send_message(msg_verack()) + if self.support_addrv2: + self.send_message(msg_sendaddrv2()) self.nServices = message.nServices # Connection helper methods diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index cc49ce92771e6..4de486415cf57 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -70,6 +70,7 @@ 'mining_pos_coldStaking.py', # ~ 220 sec 'wallet_import_rescan.py', # ~ 204 sec 'p2p_invalid_block.py', # ~ 213 sec + 'p2p_addrv2_relay.py', 'p2p_invalid_messages.py', 'feature_reindex.py', # ~ 205 sec 'feature_logging.py', # ~ 195 sec From ccd508a027be6936daad642e75e32800dbeb0341 Mon Sep 17 00:00:00 2001 From: Vasil Dimov Date: Mon, 14 Sep 2020 14:30:15 +0200 Subject: [PATCH 37/58] tor: make a TORv3 hidden service instead of TORv2 TORv2 is deprecated [1], thus whenever we create the hidden service ourselves create a TORv3 one instead. [1] https://blog.torproject.org/v2-deprecation-timeline --- doc/release-notes.md | 13 +++++++++++++ src/torcontrol.cpp | 9 +++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/doc/release-notes.md b/doc/release-notes.md index 35bd4c2d116db..827210cba55e4 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -284,6 +284,19 @@ It enforces the same rules as the legacy cold-staking opcode, but without allowi The new opcode takes the name of `OP_CHECKCOLDSTAKEVERIFY`, and the legacy opcode (`0xd1`) is renamed to `OP_CHECKCOLDSTAKEVERIFY_LOF` (last-output-free). Scripts with the old opcode are still accepted on the network (the restriction on the last-output is enforced after the script validation in this case), but the client creates new delegations with the new opcode, by default, after the upgrade enforcement. +P2P and network changes +----------------------- + +- The Tor onion service that is automatically created by setting the + `-listenonion` configuration parameter will now be created as a Tor v3 service + instead of Tor v2. The private key that was used for Tor v2 (if any) will be + left untouched in the `onion_private_key` file in the data directory (see + `-datadir`) and can be removed if not needed. PIVX Core will no longer + attempt to read it. The private key for the Tor v3 service will be saved in a + file named `onion_v3_private_key`. To use the deprecated Tor v2 service (not + recommended), then `onion_private_key` can be copied over + `onion_v3_private_key`, e.g. + `cp -f onion_private_key onion_v3_private_key`. (#19954) Multi-wallet support -------------------- diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 6bce9b322e685..4f07dbef4ad93 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -535,9 +535,10 @@ void TorController::auth_cb(TorControlConnection& _conn, const TorControlReply& } // Finally - now create the service - if (private_key.empty()) // No private key, generate one - private_key = "NEW:RSA1024"; // Explicitly request RSA1024 - see issue #9214 - // Request hidden service, redirect port. + if (private_key.empty()) { // No private key, generate one + private_key = "NEW:ED25519-V3"; // Explicitly request key type - see issue #9214 + } + // Request onion service, redirect port. // Note that the 'virtual' port is always the default port to avoid decloaking nodes using other ports. _conn.Command(strprintf("ADD_ONION %s Port=%i,127.0.0.1:%i", private_key, Params().GetDefaultPort(), GetListenPort()), std::bind(&TorController::add_onion_cb, this, std::placeholders::_1, std::placeholders::_2)); @@ -721,7 +722,7 @@ void TorController::Reconnect() fs::path TorController::GetPrivateKeyFile() { - return GetDataDir() / "onion_private_key"; + return GetDataDir() / "onion_v3_private_key"; } void TorController::reconnect_cb(evutil_socket_t fd, short what, void *arg) From 61c2ed4d8cf016ca0eb0d5d818b578abfc787110 Mon Sep 17 00:00:00 2001 From: furszy Date: Fri, 11 Jun 2021 22:46:18 -0300 Subject: [PATCH 38/58] Bump net protocol version + don't send 'sendaddrv2' to pre-70923 software --- src/net_processing.cpp | 18 ++++++++++++------ src/version.h | 2 +- test/functional/test_framework/messages.py | 2 +- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 79643c69b9d69..5358aead27a35 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1147,6 +1147,18 @@ bool static ProcessMessage(CNode* pfrom, std::string strCommand, CDataStream& vR if (pfrom->fInbound) PushNodeVersion(pfrom, connman, GetAdjustedTime()); + CNetMsgMaker msg_maker(INIT_PROTO_VERSION); + connman->PushMessage(pfrom, msg_maker.Make(NetMsgType::VERACK)); + + if (nVersion >= 70923) { + // BIP155 defines addrv2 and sendaddrv2 for all protocol versions, but some + // implementations reject messages they don't know. As a courtesy, don't send + // it to nodes with a version before 70923 (v5.2.99), as no software is known to support + // BIP155 that doesn't announce at least that protocol version number. + + connman->PushMessage(pfrom, msg_maker.Make(NetMsgType::SENDADDRV2)); + } + pfrom->nServices = nServices; pfrom->SetAddrLocal(addrMe); { @@ -1170,12 +1182,6 @@ bool static ProcessMessage(CNode* pfrom, std::string strCommand, CDataStream& vR pfrom->SetSendVersion(nSendVersion); pfrom->nVersion = nVersion; - CNetMsgMaker msg_maker(INIT_PROTO_VERSION); - connman->PushMessage(pfrom, msg_maker.Make(NetMsgType::VERACK)); - - // Signal ADDRv2 support (BIP155). - connman->PushMessage(pfrom, msg_maker.Make(NetMsgType::SENDADDRV2)); - { LOCK(cs_main); // Potentially mark this peer as a preferred download peer. diff --git a/src/version.h b/src/version.h index 030dd9554eb14..9df15fe40c0df 100644 --- a/src/version.h +++ b/src/version.h @@ -11,7 +11,7 @@ * network protocol versioning */ -static const int PROTOCOL_VERSION = 70922; +static const int PROTOCOL_VERSION = 70923; //! initial proto version, to be increased after version/verack negotiation static const int INIT_PROTO_VERSION = 209; diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 770c058cde18c..1e2408d40bdfb 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -29,7 +29,7 @@ from .util import hex_str_to_bytes, bytes_to_hex_str MIN_VERSION_SUPPORTED = 60001 -MY_VERSION = 70922 +MY_VERSION = 70923 MY_SUBVERSION = "/python-mininode-tester:0.0.3/" MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37) From ba954cabdc411439a0af3f3e3c682dcafe9353d3 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Fri, 11 Jun 2021 22:48:08 -0300 Subject: [PATCH 39/58] Send and require SENDADDRV2 before VERACK See the corresponding BIP change: bitcoin/bips#1043 --- src/net_processing.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 5358aead27a35..79e0f79f05a70 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1148,7 +1148,6 @@ bool static ProcessMessage(CNode* pfrom, std::string strCommand, CDataStream& vR PushNodeVersion(pfrom, connman, GetAdjustedTime()); CNetMsgMaker msg_maker(INIT_PROTO_VERSION); - connman->PushMessage(pfrom, msg_maker.Make(NetMsgType::VERACK)); if (nVersion >= 70923) { // BIP155 defines addrv2 and sendaddrv2 for all protocol versions, but some @@ -1159,6 +1158,8 @@ bool static ProcessMessage(CNode* pfrom, std::string strCommand, CDataStream& vR connman->PushMessage(pfrom, msg_maker.Make(NetMsgType::SENDADDRV2)); } + connman->PushMessage(pfrom, msg_maker.Make(NetMsgType::VERACK)); + pfrom->nServices = nServices; pfrom->SetAddrLocal(addrMe); { From 78aa61c3044c819b44d4a61fa0e01d4c0fb95177 Mon Sep 17 00:00:00 2001 From: furszy Date: Fri, 11 Jun 2021 23:17:09 -0300 Subject: [PATCH 40/58] net: Make addr relay mockable Adaptation of btc@fa47a0b003f53708b6d5df1ed4e7f8a7c68aa3ac --- src/net.cpp | 2 -- src/net.h | 4 ++-- src/net_processing.cpp | 8 ++++---- src/validation.h | 8 ++++---- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/net.cpp b/src/net.cpp index 16add4d89c652..b94233af0fe59 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -2311,8 +2311,6 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn filterInventoryKnown.reset(); fSendMempool = false; fGetAddr = false; - nNextLocalAddrSend = 0; - nNextAddrSend = 0; fRelayTxes = false; pfilter = std::make_unique(); timeLastMempoolReq = 0; diff --git a/src/net.h b/src/net.h index a8bafdd0eb0bd..549ef3aba159c 100644 --- a/src/net.h +++ b/src/net.h @@ -607,8 +607,8 @@ class CNode CRollingBloomFilter addrKnown; bool fGetAddr; std::set setKnown; - int64_t nNextAddrSend; - int64_t nNextLocalAddrSend; + std::chrono::microseconds m_next_addr_send GUARDED_BY(cs_sendProcessing){0}; + std::chrono::microseconds m_next_local_addr_send GUARDED_BY(cs_sendProcessing){0}; // inventory based relay CRollingBloomFilter filterInventoryKnown; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 79e0f79f05a70..0c3911e6fbbb7 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2101,16 +2101,16 @@ bool PeerLogicValidation::SendMessages(CNode* pto, std::atomic& interruptM int64_t nNow = GetTimeMicros(); auto current_time = GetTime(); - if (!IsInitialBlockDownload() && pto->nNextLocalAddrSend < nNow) { + if (!IsInitialBlockDownload() && pto->m_next_local_addr_send < current_time) { AdvertiseLocal(pto); - pto->nNextLocalAddrSend = PoissonNextSend(nNow, AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL); + pto->m_next_local_addr_send = PoissonNextSend(current_time, AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL); } // // Message: addr // - if (pto->nNextAddrSend < nNow) { - pto->nNextAddrSend = PoissonNextSend(nNow, AVG_ADDRESS_BROADCAST_INTERVAL); + if (pto->m_next_addr_send < current_time) { + pto->m_next_addr_send = PoissonNextSend(current_time, AVG_ADDRESS_BROADCAST_INTERVAL); std::vector vAddr; vAddr.reserve(pto->vAddrToSend.size()); diff --git a/src/validation.h b/src/validation.h index 17835667058d1..598c65c49a061 100644 --- a/src/validation.h +++ b/src/validation.h @@ -103,10 +103,10 @@ static const unsigned int BLOCK_DOWNLOAD_WINDOW = 1024; static const unsigned int DATABASE_WRITE_INTERVAL = 60 * 60; /** Time to wait (in seconds) between flushing chainstate to disk. */ static const unsigned int DATABASE_FLUSH_INTERVAL = 24 * 60 * 60; -/** Average delay between local address broadcasts in seconds. */ -static const unsigned int AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL = 24 * 24 * 60; -/** Average delay between peer address broadcasts in seconds. */ -static const unsigned int AVG_ADDRESS_BROADCAST_INTERVAL = 30; +/** Average delay between local address broadcasts */ +static constexpr std::chrono::hours AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL{24}; +/** Average delay between peer address broadcasts */ +static constexpr std::chrono::seconds AVG_ADDRESS_BROADCAST_INTERVAL{30}; /** Default multiplier used in the computation for shielded txes min fee */ static const unsigned int DEFAULT_SHIELDEDTXFEE_K = 100; /** Enable bloom filter */ From 6a4f1e038ae1bd433b087f1c2584f8979efed4a7 Mon Sep 17 00:00:00 2001 From: furszy Date: Thu, 1 Jul 2021 11:50:26 -0300 Subject: [PATCH 41/58] test: Add basic addr relay test Adapted version of btc@fa1da3d4bfc0511a89f5b19d5a4d89e55ff7ccde --- test/functional/p2p_addr_relay.py | 71 +++++++++++++++++++++++++++++++ test/functional/test_runner.py | 1 + 2 files changed, 72 insertions(+) create mode 100755 test/functional/p2p_addr_relay.py diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py new file mode 100755 index 0000000000000..66bb463f2374a --- /dev/null +++ b/test/functional/p2p_addr_relay.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test addr relay +""" + +import time + +from test_framework.messages import ( + CAddress, + NODE_NETWORK, + msg_addr, +) +from test_framework.mininode import ( + P2PInterface, +) +from test_framework.test_framework import PivxTestFramework +from test_framework.util import ( + assert_equal, +) + +ADDRS = [] +for i in range(10): + addr = CAddress() + addr.time = int(time.time()) + i + addr.nServices = NODE_NETWORK + addr.ip = "123.123.123.{}".format(i % 256) + addr.port = 8333 + i + ADDRS.append(addr) + + +class AddrReceiver(P2PInterface): + def on_addr(self, message): + for addr in message.addrs: + assert_equal(addr.nServices, 1) + assert addr.ip.startswith('123.123.123.') + assert (8333 <= addr.port < 8343) + + +class AddrTest(PivxTestFramework): + def set_test_params(self): + self.setup_clean_chain = False + self.num_nodes = 1 + + def run_test(self): + self.log.info('Create connection that sends addr messages') + addr_source = self.nodes[0].add_p2p_connection(P2PInterface()) + msg = msg_addr() + + self.log.info('Send too large addr message') + msg.addrs = ADDRS * 101 + with self.nodes[0].assert_debug_log(['addr message size = 1010']): + addr_source.send_and_ping(msg) + + self.log.info('Check that addr message content is relayed and added to addrman') + addr_receiver = self.nodes[0].add_p2p_connection(AddrReceiver()) + msg.addrs = ADDRS + with self.nodes[0].assert_debug_log([ + 'Added 10 addresses from 127.0.0.1: 0 tried', + 'received: addr (301 bytes) peer=1', + 'sending addr (301 bytes) peer=2', + ]): + addr_source.send_and_ping(msg) + self.nodes[0].setmocktime(int(time.time()) + 30 * 60) + addr_receiver.sync_with_ping() + + +if __name__ == '__main__': + AddrTest().main() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 4de486415cf57..5542852a76e42 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -70,6 +70,7 @@ 'mining_pos_coldStaking.py', # ~ 220 sec 'wallet_import_rescan.py', # ~ 204 sec 'p2p_invalid_block.py', # ~ 213 sec + 'p2p_addr_relay.py', 'p2p_addrv2_relay.py', 'p2p_invalid_messages.py', 'feature_reindex.py', # ~ 205 sec From 61a08afdbadf4b5213e783f04fae7c9326224f5e Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Tue, 6 Mar 2018 16:48:15 -0500 Subject: [PATCH 42/58] [tests] bind functional test nodes to 127.0.0.1 Prevents OSX firewall Allow-this-application-to-accept-inbound-connections permission popups and is generally safer. To prevent binding to 127.0.0.1, set self.bind_to_localhost_only = False. --- test/functional/rpc_bind.py | 1 + test/functional/test_framework/test_framework.py | 11 ++++++++--- test/functional/test_framework/test_node.py | 9 +++++++-- test/functional/test_framework/util.py | 6 ++++++ test/functional/wallet_dump.py | 2 +- test/functional/wallet_import_rescan.py | 2 +- 6 files changed, 24 insertions(+), 7 deletions(-) diff --git a/test/functional/rpc_bind.py b/test/functional/rpc_bind.py index 31ecdf8bbd8d8..5811ab20f265a 100755 --- a/test/functional/rpc_bind.py +++ b/test/functional/rpc_bind.py @@ -22,6 +22,7 @@ class RPCBindTest(PivxTestFramework): def set_test_params(self): self.setup_clean_chain = True + self.bind_to_localhost_only = False self.num_nodes = 1 def setup_network(self): diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 27b1532e45af1..fb5bd3e5591d8 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -100,6 +100,7 @@ def __init__(self): self.mocktime = 0 self.rpc_timewait = 600 # Wait for up to 600 seconds for the RPC server to respond self.supports_cli = False + self.bind_to_localhost_only = True self.set_test_params() assert hasattr(self, "num_nodes"), "Test must set self.num_nodes in set_test_params()" @@ -272,7 +273,10 @@ def run_test(self): def add_nodes(self, num_nodes, extra_args=None, *, rpchost=None, binary=None): """Instantiate TestNode objects""" - + if self.bind_to_localhost_only: + extra_confs = [["bind=127.0.0.1"]] * num_nodes + else: + extra_confs = [[]] * num_nodes if extra_args is None: extra_args = [[]] * num_nodes # Check wallet version @@ -282,10 +286,11 @@ def add_nodes(self, num_nodes, extra_args=None, *, rpchost=None, binary=None): self.log.info("Running test with legacy (pre-HD) wallet") if binary is None: binary = [None] * num_nodes + assert_equal(len(extra_confs), num_nodes) assert_equal(len(extra_args), num_nodes) assert_equal(len(binary), num_nodes) for i in range(num_nodes): - self.nodes.append(TestNode(i, self.options.tmpdir, extra_args[i], rpchost, timewait=self.rpc_timewait, binary=binary[i], stderr=None, mocktime=self.mocktime, coverage_dir=self.options.coveragedir, use_cli=self.options.usecli)) + self.nodes.append(TestNode(i, self.options.tmpdir, rpchost=rpchost, timewait=self.rpc_timewait, binary=binary[i], stderr=None, mocktime=self.mocktime, coverage_dir=self.options.coveragedir, extra_conf=extra_confs[i], extra_args=extra_args[i], use_cli=self.options.usecli)) def start_node(self, i, *args, **kwargs): """Start a pivxd""" @@ -531,7 +536,7 @@ def start_nodes_from_dir(ddir, num_nodes=MAX_NODES): args = [os.getenv("BITCOIND", "pivxd"), "-spendzeroconfchange=1", "-server", "-keypool=1", "-datadir=" + datadir, "-discover=0"] self.nodes.append( - TestNode(i, ddir, extra_args=[], rpchost=None, timewait=self.rpc_timewait, binary=None, stderr=None, + TestNode(i, ddir, extra_conf=["bind=127.0.0.1"], extra_args=[], rpchost=None, timewait=self.rpc_timewait, binary=None, stderr=None, mocktime=self.mocktime, coverage_dir=None)) self.nodes[i].args = args self.start_node(i) diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index c7a9ec05b18e7..bf5d9c8e1842a 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -19,6 +19,7 @@ from .authproxy import JSONRPCException from .util import ( + append_config, assert_equal, delete_cookie_file, get_rpc_proxy, @@ -53,7 +54,7 @@ class TestNode(): To make things easier for the test writer, any unrecognised messages will be dispatched to the RPC connection.""" - def __init__(self, i, dirname, extra_args, rpchost, timewait, binary, stderr, mocktime, coverage_dir, use_cli=False): + def __init__(self, i, dirname, rpchost, timewait, binary, stderr, mocktime, coverage_dir, extra_conf=None, extra_args=None, use_cli=False): self.index = i self.datadir = os.path.join(dirname, "node" + str(i)) self.rpchost = rpchost @@ -64,7 +65,11 @@ def __init__(self, i, dirname, extra_args, rpchost, timewait, binary, stderr, mo self.binary = binary self.stderr = stderr self.coverage_dir = coverage_dir - # Most callers will just need to add extra args to the standard list below. For those callers that need more flexibity, they can just set the args property directly. + if extra_conf is not None: + append_config(dirname, i, extra_conf) + # Most callers will just need to add extra args to the standard list below. + # For those callers that need more flexibity, they can just set the args property directly. + # Note that common args are set in the config file (see initialize_datadir) self.extra_args = extra_args self.args = [ self.binary, diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 2e223b1918815..e08cd23b0f5e2 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -322,6 +322,12 @@ def initialize_datadir(dirname, n): def get_datadir_path(dirname, n): return os.path.join(dirname, "node" + str(n)) +def append_config(dirname, n, options): + datadir = get_datadir_path(dirname, n) + with open(os.path.join(datadir, "pivx.conf"), 'a', encoding='utf8') as f: + for option in options: + f.write(option + "\n") + def get_auth_cookie(datadir): user = None password = None diff --git a/test/functional/wallet_dump.py b/test/functional/wallet_dump.py index 749f26ce4e64b..ab830a5fa5f60 100755 --- a/test/functional/wallet_dump.py +++ b/test/functional/wallet_dump.py @@ -79,7 +79,7 @@ def setup_network(self, split=False): # longer than the default 30 seconds due to an expensive # CWallet::TopUpKeyPool call, and the encryptwallet RPC made later in # the test often takes even longer. - self.add_nodes(self.num_nodes, self.extra_args) + self.add_nodes(self.num_nodes, extra_args=self.extra_args) self.start_nodes() def run_test (self): diff --git a/test/functional/wallet_import_rescan.py b/test/functional/wallet_import_rescan.py index 46e6545e08d55..d7c5a5b5d03dd 100755 --- a/test/functional/wallet_import_rescan.py +++ b/test/functional/wallet_import_rescan.py @@ -123,7 +123,7 @@ def setup_network(self): if import_node.prune: extra_args[i] += ["-prune=1"] - self.add_nodes(self.num_nodes, extra_args) + self.add_nodes(self.num_nodes, extra_args=extra_args) self.start_nodes() for i in range(1, self.num_nodes): connect_nodes(self.nodes[i], 0) From 4a034d85848c2769a439e62533bf163c2fb14110 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Thu, 22 Feb 2018 15:43:26 +0100 Subject: [PATCH 43/58] test: Add rpc_bind test to default-run tests Skip the parts that cannot be run on the host due to lack of IPv6 support or a second interface to bind on, and warn appropriately. Without no strong requirements (besides being Linux only, in which case the test is skipped) left, just add this test to the default in test_runner. Includes suggested changes by John Newbery. --- test/functional/rpc_bind.py | 104 +++++++++++++++++++-------------- test/functional/test_runner.py | 4 +- 2 files changed, 64 insertions(+), 44 deletions(-) diff --git a/test/functional/rpc_bind.py b/test/functional/rpc_bind.py index 5811ab20f265a..16d51d6236dfa 100755 --- a/test/functional/rpc_bind.py +++ b/test/functional/rpc_bind.py @@ -4,10 +4,9 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test running pivxd with the -rpcbind and -rpcallowip options.""" -import socket import sys -from test_framework.netutil import addr_to_hex, all_interfaces, get_bind_addrs +from test_framework.netutil import addr_to_hex, all_interfaces, get_bind_addrs, test_ipv6_local from test_framework.test_framework import PivxTestFramework, SkipTest from test_framework.util import ( assert_equal, @@ -28,6 +27,11 @@ def set_test_params(self): def setup_network(self): self.add_nodes(self.num_nodes, None) + def add_options(self, parser): + parser.add_option("--ipv4", action='store_true', dest="run_ipv4", help="Run ipv4 tests only", default=False) + parser.add_option("--ipv6", action='store_true', dest="run_ipv6", help="Run ipv6 tests only", default=False) + parser.add_option("--nonloopback", action='store_true', dest="run_nonloopback", help="Run non-loopback tests only", default=False) + def run_bind_test(self, allow_ips, connect_to, addresses, expected): ''' Start a node with requested rpcallowip and rpcbind parameters, @@ -62,55 +66,69 @@ def run_allowip_test(self, allow_ips, rpchost, rpcport): def run_test(self): # due to OS-specific network stats queries, this test works only on Linux + if sum([self.options.run_ipv4, self.options.run_ipv6, self.options.run_nonloopback]) > 1: + raise AssertionError("Only one of --ipv4, --ipv6 and --nonloopback can be set") + + self.log.info("Check for linux") if not sys.platform.startswith('linux'): raise SkipTest("This test can only be run on linux.") - # find the first non-loopback interface for testing - non_loopback_ip = None + + self.log.info("Check for ipv6") + have_ipv6 = test_ipv6_local() + if not have_ipv6 and not self.options.run_ipv4: + raise SkipTest("This test requires ipv6 support.") + + self.log.info("Check for non-loopback interface") + self.non_loopback_ip = None for name,ip in all_interfaces(): if ip != '127.0.0.1': - non_loopback_ip = ip + self.non_loopback_ip = ip break - if non_loopback_ip is None: - raise SkipTest("This test requires at least one non-loopback IPv4 interface.") - try: - s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) - s.connect(("::1",1)) - s.close - except OSError: - raise SkipTest("This test requires IPv6 support.") - - self.log.info("Using interface %s for testing" % non_loopback_ip) - - defaultport = rpc_port(0) - - # check default without rpcallowip (IPv4 and IPv6 localhost) - self.run_bind_test(None, '127.0.0.1', [], - [('127.0.0.1', defaultport), ('::1', defaultport)]) - # check default with rpcallowip (IPv6 any) - self.run_bind_test(['127.0.0.1'], '127.0.0.1', [], - [('::0', defaultport)]) - # check only IPv4 localhost (explicit) - self.run_bind_test(['127.0.0.1'], '127.0.0.1', ['127.0.0.1'], - [('127.0.0.1', defaultport)]) - # check only IPv4 localhost (explicit) with alternative port - self.run_bind_test(['127.0.0.1'], '127.0.0.1:32171', ['127.0.0.1:32171'], - [('127.0.0.1', 32171)]) - # check only IPv4 localhost (explicit) with multiple alternative ports on same host - self.run_bind_test(['127.0.0.1'], '127.0.0.1:32171', ['127.0.0.1:32171', '127.0.0.1:32172'], - [('127.0.0.1', 32171), ('127.0.0.1', 32172)]) - # check only IPv6 localhost (explicit) - self.run_bind_test(['[::1]'], '[::1]', ['[::1]'], - [('::1', defaultport)]) - # check both IPv4 and IPv6 localhost (explicit) - self.run_bind_test(['127.0.0.1'], '127.0.0.1', ['127.0.0.1', '[::1]'], - [('127.0.0.1', defaultport), ('::1', defaultport)]) + if self.non_loopback_ip is None and self.options.run_nonloopback: + raise SkipTest("This test requires a non-loopback ip address.") + + self.defaultport = rpc_port(0) + + if not self.options.run_nonloopback: + self._run_loopback_tests() + if not self.options.run_ipv4 and not self.options.run_ipv6: + self._run_nonloopback_tests() + + def _run_loopback_tests(self): + if self.options.run_ipv4: + # check only IPv4 localhost (explicit) + self.run_bind_test(['127.0.0.1'], '127.0.0.1', ['127.0.0.1'], + [('127.0.0.1', self.defaultport)]) + # check only IPv4 localhost (explicit) with alternative port + self.run_bind_test(['127.0.0.1'], '127.0.0.1:32171', ['127.0.0.1:32171'], + [('127.0.0.1', 32171)]) + # check only IPv4 localhost (explicit) with multiple alternative ports on same host + self.run_bind_test(['127.0.0.1'], '127.0.0.1:32171', ['127.0.0.1:32171', '127.0.0.1:32172'], + [('127.0.0.1', 32171), ('127.0.0.1', 32172)]) + else: + # check default without rpcallowip (IPv4 and IPv6 localhost) + self.run_bind_test(None, '127.0.0.1', [], + [('127.0.0.1', self.defaultport), ('::1', self.defaultport)]) + # check default with rpcallowip (IPv6 any) + self.run_bind_test(['127.0.0.1'], '127.0.0.1', [], + [('::0', self.defaultport)]) + # check only IPv6 localhost (explicit) + self.run_bind_test(['[::1]'], '[::1]', ['[::1]'], + [('::1', self.defaultport)]) + # check both IPv4 and IPv6 localhost (explicit) + self.run_bind_test(['127.0.0.1'], '127.0.0.1', ['127.0.0.1', '[::1]'], + [('127.0.0.1', self.defaultport), ('::1', self.defaultport)]) + + def _run_nonloopback_tests(self): + self.log.info("Using interface %s for testing" % self.non_loopback_ip) + # check only non-loopback interface - self.run_bind_test([non_loopback_ip], non_loopback_ip, [non_loopback_ip], - [(non_loopback_ip, defaultport)]) + self.run_bind_test([self.non_loopback_ip], self.non_loopback_ip, [self.non_loopback_ip], + [(self.non_loopback_ip, self.defaultport)]) # Check that with invalid rpcallowip, we are denied - self.run_allowip_test([non_loopback_ip], non_loopback_ip, defaultport) - assert_raises_rpc_error(-342, "non-JSON HTTP response with '403 Forbidden' from server", self.run_allowip_test, ['1.1.1.1'], non_loopback_ip, defaultport) + self.run_allowip_test([self.non_loopback_ip], self.non_loopback_ip, self.defaultport) + assert_raises_rpc_error(-342, "non-JSON HTTP response with '403 Forbidden' from server", self.run_allowip_test, ['1.1.1.1'], self.non_loopback_ip, self.defaultport) if __name__ == '__main__': RPCBindTest().main() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 5542852a76e42..c81200eba6ac3 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -69,6 +69,9 @@ 'rpc_fundrawtransaction.py', # ~ 227 sec 'mining_pos_coldStaking.py', # ~ 220 sec 'wallet_import_rescan.py', # ~ 204 sec + 'rpc_bind.py --ipv4', + 'rpc_bind.py --ipv6', + 'rpc_bind.py --nonloopback', 'p2p_invalid_block.py', # ~ 213 sec 'p2p_addr_relay.py', 'p2p_addrv2_relay.py', @@ -187,7 +190,6 @@ # vv Tests less than 60s vv #'p2p_feefilter.py', 'feature_abortnode.py', - 'rpc_bind.py', # vv Tests less than 30s vv #'example_test.py', 'feature_notifications.py', From d4d6729c51dcd0882533e69947b6ff63e43909b2 Mon Sep 17 00:00:00 2001 From: Kristaps Kaupe Date: Thu, 22 Nov 2018 23:57:07 +0200 Subject: [PATCH 44/58] Allow running rpc_bind.py --nonloopback test without IPv6 --- test/functional/rpc_bind.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/rpc_bind.py b/test/functional/rpc_bind.py index 16d51d6236dfa..a1386cc3fc983 100755 --- a/test/functional/rpc_bind.py +++ b/test/functional/rpc_bind.py @@ -75,7 +75,7 @@ def run_test(self): self.log.info("Check for ipv6") have_ipv6 = test_ipv6_local() - if not have_ipv6 and not self.options.run_ipv4: + if not have_ipv6 and not (self.options.run_ipv4 or self.options.run_nonloopback): raise SkipTest("This test requires ipv6 support.") self.log.info("Check for non-loopback interface") From 647d60b69e6badcfd3b495f17b7728e94db07049 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Mon, 3 Dec 2018 09:15:07 -0800 Subject: [PATCH 45/58] tests: Modify rpc_bind to conform to #14532 behaviour. - Even when rpcallowip is specified, only bind localhost - Explicitly bind in run_allowip_test --- test/functional/rpc_bind.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/functional/rpc_bind.py b/test/functional/rpc_bind.py index a1386cc3fc983..b77d3eba9bea1 100755 --- a/test/functional/rpc_bind.py +++ b/test/functional/rpc_bind.py @@ -56,9 +56,12 @@ def run_allowip_test(self, allow_ips, rpchost, rpcport): at a non-localhost IP. ''' self.log.info("Allow IP test for %s:%d" % (rpchost, rpcport)) - base_args = ['-disablewallet', '-nolisten'] + ['-rpcallowip='+x for x in allow_ips] + node_args = \ + ['-disablewallet', '-nolisten'] + \ + ['-rpcallowip='+x for x in allow_ips] + \ + ['-rpcbind='+addr for addr in ['127.0.0.1', "%s:%d" % (rpchost, rpcport)]] # Bind to localhost as well so start_nodes doesn't hang self.nodes[0].rpchost = None - self.start_nodes([base_args]) + self.start_nodes([node_args]) # connect to node through non-loopback interface node = get_rpc_proxy(rpc_url(get_datadir_path(self.options.tmpdir, 0), 0, "%s:%d" % (rpchost, rpcport)), 0, coveragedir=self.options.coveragedir) node.getnetworkinfo() @@ -109,9 +112,9 @@ def _run_loopback_tests(self): # check default without rpcallowip (IPv4 and IPv6 localhost) self.run_bind_test(None, '127.0.0.1', [], [('127.0.0.1', self.defaultport), ('::1', self.defaultport)]) - # check default with rpcallowip (IPv6 any) + # check default with rpcallowip (IPv4 and IPv6 localhost) self.run_bind_test(['127.0.0.1'], '127.0.0.1', [], - [('::0', self.defaultport)]) + [('127.0.0.1', self.defaultport), ('::1', self.defaultport)]) # check only IPv6 localhost (explicit) self.run_bind_test(['[::1]'], '[::1]', ['[::1]'], [('::1', self.defaultport)]) From 57fc7b090300066bbcce1c31864ce5e27c64b31a Mon Sep 17 00:00:00 2001 From: Jon Atack Date: Mon, 12 Oct 2020 09:31:08 +0200 Subject: [PATCH 46/58] net: update GetNetworkName() with all enum Network cases --- src/netbase.cpp | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/netbase.cpp b/src/netbase.cpp index 9004540464aca..5d54fe4bdae3c 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -50,15 +50,17 @@ enum Network ParseNetwork(std::string net) std::string GetNetworkName(enum Network net) { switch (net) { - case NET_IPV4: - return "ipv4"; - case NET_IPV6: - return "ipv6"; - case NET_ONION: - return "onion"; - default: - return ""; - } + case NET_UNROUTABLE: return "unroutable"; + case NET_IPV4: return "ipv4"; + case NET_IPV6: return "ipv6"; + case NET_ONION: return "onion"; + case NET_I2P: return "i2p"; + case NET_CJDNS: return "cjdns"; + case NET_INTERNAL: return "internal"; + case NET_MAX: assert(false); + } // no default case, so the compiler can warn about missing cases + + assert(false); } void SplitHostPort(std::string in, int& portOut, std::string& hostOut) From d8e01b507a47032797295b6c01e035fef1ee9861 Mon Sep 17 00:00:00 2001 From: Jon Atack Date: Sun, 11 Oct 2020 11:49:08 +0200 Subject: [PATCH 47/58] rpc: update GetNetworksInfo() to not return unsupported networks --- src/rpc/net.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 2479d58a189fa..f392bf3c3ccc3 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -349,8 +349,7 @@ static UniValue GetNetworksInfo() UniValue networks(UniValue::VARR); for (int n = 0; n < NET_MAX; ++n) { enum Network network = static_cast(n); - if (network == NET_UNROUTABLE || network == NET_INTERNAL) - continue; + if (network == NET_UNROUTABLE || network == NET_I2P || network == NET_CJDNS || network == NET_INTERNAL) continue; proxyType proxy; UniValue obj(UniValue::VOBJ); GetProxy(network, proxy); From bb90c5c97e34eb75c70fa6731e014e900157f20d Mon Sep 17 00:00:00 2001 From: Jon Atack Date: Sun, 11 Oct 2020 11:49:25 +0200 Subject: [PATCH 48/58] test: add getnetworkinfo network name regression tests --- test/functional/feature_proxy.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/test/functional/feature_proxy.py b/test/functional/feature_proxy.py index c0a2c1da87ec3..134d5571dd54c 100755 --- a/test/functional/feature_proxy.py +++ b/test/functional/feature_proxy.py @@ -25,6 +25,8 @@ addnode connect to IPv6 addnode connect to onion addnode connect to generic DNS name + +- Test getnetworkinfo for each node """ import os @@ -42,6 +44,16 @@ RANGE_BEGIN = PORT_MIN + 2 * PORT_RANGE # Start after p2p and rpc ports +# Networks returned by RPC getpeerinfo, defined in src/netbase.cpp::GetNetworkName() +NET_UNROUTABLE = "unroutable" +NET_IPV4 = "ipv4" +NET_IPV6 = "ipv6" +NET_ONION = "onion" + +# Networks returned by RPC getnetworkinfo, defined in src/rpc/net.cpp::GetNetworksInfo() +NETWORKS = frozenset({NET_IPV4, NET_IPV6, NET_ONION}) + + class ProxyTest(PivxTestFramework): def set_test_params(self): self.num_nodes = 4 @@ -77,14 +89,14 @@ def setup_nodes(self): self.serv3 = Socks5Server(self.conf3) self.serv3.start() - # Note: proxies are not used to connect to local nodes - # this is because the proxy to use is based on CService.GetNetwork(), which return NET_UNROUTABLE for localhost + # Note: proxies are not used to connect to local nodes. This is because the proxy to + # use is based on CService.GetNetwork(), which returns NET_UNROUTABLE for localhost. args = [ ['-listen', '-proxy=%s:%i' % (self.conf1.addr),'-proxyrandomize=1'], ['-listen', '-proxy=%s:%i' % (self.conf1.addr),'-onion=%s:%i' % (self.conf2.addr),'-proxyrandomize=0'], ['-listen', '-proxy=%s:%i' % (self.conf2.addr),'-proxyrandomize=1'], [] - ] + ] if self.have_ipv6: args[3] = ['-listen', '-proxy=[%s]:%i' % (self.conf3.addr),'-proxyrandomize=0', '-noonion'] self.add_nodes(self.num_nodes, extra_args=args) @@ -169,15 +181,17 @@ def networks_dict(d): r[x['name']] = x return r - # test RPC getnetworkinfo + self.log.info("Test RPC getnetworkinfo") n0 = networks_dict(self.nodes[0].getnetworkinfo()) - for net in ['ipv4','ipv6','onion']: + assert_equal(NETWORKS, n0.keys()) + for net in NETWORKS: assert_equal(n0[net]['proxy'], '%s:%i' % (self.conf1.addr)) assert_equal(n0[net]['proxy_randomize_credentials'], True) assert_equal(n0['onion']['reachable'], True) n1 = networks_dict(self.nodes[1].getnetworkinfo()) - for net in ['ipv4','ipv6']: + assert_equal(NETWORKS, n1.keys()) + for net in ['ipv4', 'ipv6']: assert_equal(n1[net]['proxy'], '%s:%i' % (self.conf1.addr)) assert_equal(n1[net]['proxy_randomize_credentials'], False) assert_equal(n1['onion']['proxy'], '%s:%i' % (self.conf2.addr)) @@ -185,14 +199,16 @@ def networks_dict(d): assert_equal(n1['onion']['reachable'], True) n2 = networks_dict(self.nodes[2].getnetworkinfo()) - for net in ['ipv4','ipv6','onion']: + assert_equal(NETWORKS, n2.keys()) + for net in NETWORKS: assert_equal(n2[net]['proxy'], '%s:%i' % (self.conf2.addr)) assert_equal(n2[net]['proxy_randomize_credentials'], True) assert_equal(n2['onion']['reachable'], True) if self.have_ipv6: n3 = networks_dict(self.nodes[3].getnetworkinfo()) - for net in ['ipv4','ipv6']: + assert_equal(NETWORKS, n3.keys()) + for net in NETWORKS: assert_equal(n3[net]['proxy'], '[%s]:%i' % (self.conf3.addr)) assert_equal(n3[net]['proxy_randomize_credentials'], False) assert_equal(n3['onion']['reachable'], False) From 89df7f2b007dfafa5abfbd894cef7e722717a4e2 Mon Sep 17 00:00:00 2001 From: Vasil Dimov Date: Mon, 2 Nov 2020 11:36:29 +0100 Subject: [PATCH 49/58] addrman: ensure old versions don't parse peers.dat Even though the format of `peers.dat` was changed in an incompatible way (old software versions <0.21 cannot understand the new file format), it is not guaranteed that old versions will fail to parse it. There is a chance that old versions parse its contents as garbage and use it. Old versions expect the "key size" field to be 32 and fail the parsing if it is not. Thus, we put something other than 32 in it. This will make versions between 0.11.0 and 0.20.1 deterministically fail on the new format. Versions prior to https://github.com/bitcoin/bitcoin/pull/5941 (<0.11.0) will still parse it as garbage. Also, introduce a way to increment the `peers.dat` format in a way that does not necessary make older versions refuse to read it. --- doc/release-notes.md | 4 +-- src/addrman.h | 81 ++++++++++++++++++++++++++++++-------------- 2 files changed, 58 insertions(+), 27 deletions(-) diff --git a/doc/release-notes.md b/doc/release-notes.md index 827210cba55e4..0a8b7c794cc7d 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -43,8 +43,8 @@ format of this file has been changed in a backwards-incompatible way in order to accommodate the storage of Tor v3 and other BIP155 addresses. This means that if the file is modified by v5.3 or newer then older versions will not be able to read it. Those old versions, in the event of a downgrade, will log an error -message that deserialization has failed and will continue normal operation -as if the file was missing, creating a new empty one. (#2411) +message "Incorrect keysize in addrman deserialization" and will continue normal +operation as if the file was missing, creating a new empty one. (#2411) Notable Changes ============== diff --git a/src/addrman.h b/src/addrman.h index f272e9b23714d..88b04bce64778 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -7,6 +7,10 @@ #ifndef BITCOIN_ADDRMAN_H #define BITCOIN_ADDRMAN_H +#if defined(HAVE_CONFIG_H) +#include "config/pivx-config.h" +#endif //HAVE_CONFIG_H + #include "clientversion.h" #include "netaddress.h" #include "protocol.h" @@ -183,6 +187,28 @@ friend class CAddrManTest; mutable RecursiveMutex cs; private: + //! Serialization versions. + enum Format : uint8_t { + V0_HISTORICAL = 0, //!< historic format, before commit e6b343d88 + V1_DETERMINISTIC = 1, //!< for pre-asmap files + V2_ASMAP = 2, //!< for files including asmap version + V3_BIP155 = 3, //!< same as V2_ASMAP plus addresses are in BIP155 format + }; + + //! The maximum format this software knows it can unserialize. Also, we always serialize + //! in this format. + //! The format (first byte in the serialized stream) can be higher than this and + //! still this software may be able to unserialize the file - if the second byte + //! (see `lowest_compatible` in `Unserialize()`) is less or equal to this. + static constexpr Format FILE_FORMAT = Format::V3_BIP155; + + //! The initial value of a field that is incremented every time an incompatible format + //! change is made (such that old software versions would not be able to parse and + //! understand the new file format). This is 32 because we overtook the "key size" + //! field which was 32 historically. + //! @note Don't increment this. Increment `lowest_compatible` in `Serialize()` instead. + static constexpr uint8_t INCOMPATIBILITY_BASE = 32; + //! last used nId int nIdCount GUARDED_BY(cs); @@ -272,14 +298,6 @@ friend class CAddrManTest; void SetServices_(const CService& addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs); public: - //! Serialization versions. - enum class Format : uint8_t { - V0_HISTORICAL = 0, //!< historic format, before commit e6b343d88 - V1_DETERMINISTIC = 1, //!< for pre-asmap files - V2_ASMAP = 2, //!< for files including asmap version - V3_BIP155 = 3, //!< same as V2_ASMAP plus addresses are in BIP155 format - }; - // Compressed IP->ASN mapping, loaded from a file when a node starts. // Should be always empty if no file was provided. // This mapping is then used for bucketing nodes in Addrman. @@ -302,8 +320,18 @@ friend class CAddrManTest; /** * Serialized format. - * * version byte (@see `Format`) - * * 0x20 + nKey (serialized as if it were a vector, for backward compatibility) + * * format version byte (@see `Format`) + * * lowest compatible format version byte. This is used to help old software decide + * whether to parse the file. For example: + * * PIVX Core version N knows how to parse up to format=3. If a new format=4 is + * introduced in version N+1 that is compatible with format=3 and it is known that + * version N will be able to parse it, then version N+1 will write + * (format=4, lowest_compatible=3) in the first two bytes of the file, and so + * version N will still try to parse it. + * * PIVX Core version N+2 introduces a new incompatible format=5. It will write + * (format=5, lowest_compatible=5) and so any versions that do not know how to parse + * format=5 will not try to read the file. + * * nKey * * nNew * * nTried * * number of "new" buckets XOR 2**30 @@ -334,12 +362,17 @@ friend class CAddrManTest; { LOCK(cs); - // Always serialize in the latest version (currently Format::V3_BIP155). + // Always serialize in the latest version (FILE_FORMAT). OverrideStream s(&s_, s_.GetType(), s_.GetVersion() | ADDRV2_FORMAT); - s << static_cast(Format::V3_BIP155); - s << ((unsigned char)32); + s << static_cast(FILE_FORMAT); + + // Increment `lowest_compatible` if a newly introduced format is incompatible with + // the previous one. + static constexpr uint8_t lowest_compatible = Format::V3_BIP155; + s << static_cast(INCOMPATIBILITY_BASE + lowest_compatible); + s << nKey; s << nNew; s << nTried; @@ -399,15 +432,6 @@ friend class CAddrManTest; Format format; s_ >> Using>(format); - static constexpr Format maximum_supported_format = Format::V3_BIP155; - if (format > maximum_supported_format) { - throw std::ios_base::failure(strprintf( - "Unsupported format of addrman database: %u. Maximum supported is %u. " - "Continuing operation without using the saved list of peers.", - static_cast(format), - static_cast(maximum_supported_format))); - } - int stream_version = s_.GetVersion(); if (format >= Format::V3_BIP155) { // Add ADDRV2_FORMAT to the version so that the CNetAddr and CAddress @@ -417,9 +441,16 @@ friend class CAddrManTest; OverrideStream s(&s_, s_.GetType(), stream_version); - unsigned char nKeySize; - s >> nKeySize; - if (nKeySize != 32) throw std::ios_base::failure("Incorrect keysize in addrman deserialization"); + uint8_t compat; + s >> compat; + const uint8_t lowest_compatible = compat - INCOMPATIBILITY_BASE; + if (lowest_compatible > FILE_FORMAT) { + throw std::ios_base::failure(strprintf( + "Unsupported format of addrman database: %u. It is compatible with formats >=%u, " + "but the maximum supported by this version of %s is %u.", + format, lowest_compatible, PACKAGE_NAME, static_cast(FILE_FORMAT))); + } + s >> nKey; s >> nNew; s >> nTried; From 0b5f4062488cdad27d55ddbdf4253bed6b116a4d Mon Sep 17 00:00:00 2001 From: furszy Date: Thu, 29 Jul 2021 19:24:33 -0300 Subject: [PATCH 50/58] Doc: update tor.md with latest upstream information. Combination of upstream's #19961, #21753 and #22172 --- doc/tor.md | 265 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 178 insertions(+), 87 deletions(-) diff --git a/doc/tor.md b/doc/tor.md index 773082f3d684e..5b48a4504fd4a 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -1,134 +1,225 @@ # TOR SUPPORT IN PIVX -It is possible to run PIVX Core as a Tor hidden service, and connect to such services. +It is possible to run PIVX Core as a Tor onion service, and connect to such services. The following directions assume you have a Tor proxy running on port 9050. Many distributions default to having a SOCKS proxy listening on port 9050, but others may not. In particular, the Tor Browser Bundle defaults to listening on port 9150. See [Tor Project FAQ:TBBSocksPort](https://www.torproject.org/docs/faq.html.en#TBBSocksPort) for how to properly configure Tor. +## Compatibility + +- Starting with version 5.3.0, PIVX Core only supports Tor version 3 hidden + services (Tor v3). Tor v2 addresses are ignored by PIVX Core and neither + relayed nor stored. + +- Tor removed v2 support beginning with version 0.4.6. + +## How to see information about your Tor configuration via PIVX Core + +There are several ways to see your local onion address in PIVX Core: +- in the debug log (grep for "tor:" or "AddLocal") +- in the output of RPC `getnetworkinfo` in the "localaddresses" section +- in the output of the CLI `-netinfo` peer connections dashboard + +You may set the `-debug=tor` config logging option to have additional +information in the debug log about your Tor configuration. + +CLI `-addrinfo` returns the number of addresses known to your node per network +type, including Tor v2 and v3. This is useful to see how many onion addresses +are known to your node for `-onlynet=onion` and how many Tor v3 addresses it +knows when upgrading to PIVX Core v5.3.0 and up that supports Tor v3 only. ## 1. Run PIVX Core behind a Tor proxy ----------------------------------- -The first step is running PIVX behind a Tor proxy. This will already anonymize all +The first step is running PIVX Core behind a Tor proxy. This will already anonymize all outgoing connections, but more is possible. - -proxy=ip:port Set the proxy server. If SOCKS5 is selected (default), this proxy - server will be used to try to reach .onion addresses as well. + -proxy=ip:port Set the proxy server. If SOCKS5 is selected (default), this proxy + server will be used to try to reach .onion addresses as well. + You need to use -noonion or -onion=0 to explicitly disable + outbound access to onion services. + + -onion=ip:port Set the proxy server to use for Tor onion services. You do not + need to set this if it's the same as -proxy. You can use -onion=0 + to explicitly disable access to onion services. + Note: Only the -proxy option sets the proxy for DNS requests; + with -onion they will not route over Tor, so use -proxy if you + have privacy concerns. + + -listen When using -proxy, listening is disabled by default. If you want + to manually configure an onion service (see section 3), you'll + need to enable it explicitly. + + -connect=X When behind a Tor proxy, you can specify .onion addresses instead + -addnode=X of IP addresses or hostnames in these parameters. It requires + -seednode=X SOCKS5. In Tor mode, such addresses can also be exchanged with + other P2P nodes. + + -onlynet=onion Make outgoing connections only to .onion addresses. Incoming + connections are not affected by this option. This option can be + specified multiple times to allow multiple network types, e.g. + ipv4, ipv6 or onion. If you use this option with values other + than onion you *cannot* disable onion connections; outgoing onion + connections will be enabled when you use -proxy or -onion. Use + -noonion or -onion=0 if you want to be sure there are no outbound + onion connections over the default proxy or your defined -proxy. + +In a typical situation, this suffices to run behind a Tor proxy: - -onion=ip:port Set the proxy server to use for Tor hidden services. You do not - need to set this if it's the same as -proxy. You can use -noonion - to explicitly disable access to hidden services. + ./pivxd -proxy=127.0.0.1:9050 - -listen When using -proxy, listening is disabled by default. If you want - to run a hidden service (see next section), you'll need to enable - it explicitly. +## 2. Automatically create a PIVX Core onion service - -connect=X When behind a Tor proxy, you can specify .onion addresses instead - -addnode=X of IP addresses or hostnames in these parameters. It requires - -seednode=X SOCKS5. In Tor mode, such addresses can also be exchanged with - other P2P nodes. +PIVX Core makes use of Tor's control socket API to create and destroy +ephemeral onion services programmatically. This means that if Tor is running and +proper authentication has been configured, PIVX Core automatically creates an +onion service to listen on. The goal is to increase the number of available +onion nodes. - -onlynet=onion Make outgoing connections only to .onion addresses. Incoming - connections are not affected by this option. This option can be - specified multiple times to allow multiple network types, e.g. - ipv4, ipv6, or onion. +This feature is enabled by default if PIVX Core is listening (`-listen`) and +it requires a Tor connection to work. It can be explicitly disabled with +`-listenonion=0`. If it is not disabled, it can be configured using the +`-torcontrol` and `-torpassword` settings. -In a typical situation, this suffices to run behind a Tor proxy: +To see verbose Tor information in the pivxd debug log, pass `-debug=tor`. + +### Control Port + +You may need to set up the Tor Control Port. On Linux distributions there may be +some or all of the following settings in `/etc/tor/torrc`, generally commented +out by default (if not, add them): + +``` +ControlPort 9051 +CookieAuthentication 1 +CookieAuthFileGroupReadable 1 +``` + +Add or uncomment those, save, and restart Tor (usually `systemctl restart tor` +or `sudo systemctl restart tor` on most systemd-based systems, including recent +Debian and Ubuntu, or just restart the computer). + +On some systems (such as Arch Linux), you may also need to add the following +line: + +``` +DataDirectoryGroupReadable 1 +``` + +### Authentication + +Connecting to Tor's control socket API requires one of two authentication +methods to be configured: cookie authentication or pivxd's `-torpassword` +configuration option. + +#### Cookie authentication - ./pivxd -proxy=127.0.0.1:9050 +For cookie authentication, the user running pivxd must have read access to +the `CookieAuthFile` specified in the Tor configuration. In some cases this is +preconfigured and the creation of an onion service is automatic. Don't forget to +use the `-debug=tor` pivxd configuration option to enable Tor debug logging. +If a permissions problem is seen in the debug log, e.g. `tor: Authentication +cookie /run/tor/control.authcookie could not be opened (check permissions)`, it +can be resolved by adding both the user running Tor and the user running +pivxd to the same Tor group and setting permissions appropriately. -## 2. Run a PIVX Core hidden server +On Debian-derived systems, the Tor group will likely be `debian-tor` and one way +to verify could be to list the groups and grep for a "tor" group name: -If you configure your Tor system accordingly, it is possible to make your node also -reachable from the Tor network. Add these lines to your /etc/tor/torrc (or equivalent -config file): *Needed for Tor version 0.2.7.0 and older versions of Tor only. For newer -versions of Tor see [Section 3](#3-automatically-listen-on-tor).* +``` +getent group | cut -d: -f1 | grep -i tor +``` - HiddenServiceDir /var/lib/tor/pivx-service/ - HiddenServiceVersion 2 - HiddenServicePort 51472 127.0.0.1:51472 - HiddenServicePort 61472 127.0.0.1:61472 +You can also check the group of the cookie file. On most Linux systems, the Tor +auth cookie will usually be `/run/tor/control.authcookie`: -The directory can be different of course, but (both) port numbers should be equal to -your pivxd's P2P listen port (51472 by default). +``` +stat -c '%G' /run/tor/control.authcookie +``` - -externalip=X You can tell pivx about its publicly reachable address using - this option, and this can be a v2 .onion address (v3 .onion - addresses are not supported by the PIVX network). Given the above - configuration, you can find your .onion address in - /var/lib/tor/pivx-service/hostname. For connections - coming from unroutable addresses (such as 127.0.0.1, where the - Tor proxy typically runs), .onion addresses are given - preference for your node to advertise itself with. +Once you have determined the `${TORGROUP}` and selected the `${USER}` that will +run pivxd, run this as root: - -listen You'll need to enable listening for incoming connections, as this - is off by default behind a proxy. +``` +usermod -a -G ${TORGROUP} ${USER} +``` - -discover When -externalip is specified, no attempt is made to discover local - IPv4 or IPv6 addresses. If you want to run a dual stack, reachable - from both Tor and IPv4 (or IPv6), you'll need to either pass your - other addresses using -externalip, or explicitly enable -discover. - Note that both addresses of a dual-stack system may be easily - linkable using traffic analysis. +Then restart the computer (or log out) and log in as the `${USER}` that will run +pivxd. + +#### `torpassword` authentication + +For the `-torpassword=password` option, the password is the clear text form that +was used when generating the hashed password for the `HashedControlPassword` +option in the Tor configuration file. + +The hashed password can be obtained with the command `tor --hash-password +password` (refer to the [Tor Dev +Manual](https://2019.www.torproject.org/docs/tor-manual.html.en) for more +details). + + +## 3. Manually create a PIVX Core onion service + +You can also manually configure your node to be reachable from the Tor network. +Add these lines to your `/etc/tor/torrc` (or equivalent config file): + + HiddenServiceDir /var/lib/tor/pivx-service/ + HiddenServicePort 51472 127.0.0.1:51472 + +The directory can be different of course, but virtual port numbers should be equal to +your pivxd's P2P listen port (51472 by default), and target addresses and ports +should be equal to binding address and port for inbound Tor connections (127.0.0.1:51472 by default). + + -externalip=X You can tell pivx about its publicly reachable addresses using + this option, and this can be an onion address. Given the above + configuration, you can find your onion address in + /var/lib/tor/pivx-service/hostname. For connections + coming from unroutable addresses (such as 127.0.0.1, where the + Tor proxy typically runs), onion addresses are given + preference for your node to advertise itself with. + + You can set multiple local addresses with -externalip. The + one that will be rumoured to a particular peer is the most + compatible one and also using heuristics, e.g. the address + with the most incoming connections, etc. + + -listen You'll need to enable listening for incoming connections, as this + is off by default behind a proxy. + + -discover When -externalip is specified, no attempt is made to discover local + IPv4 or IPv6 addresses. If you want to run a dual stack, reachable + from both Tor and IPv4 (or IPv6), you'll need to either pass your + other addresses using -externalip, or explicitly enable -discover. + Note that both addresses of a dual-stack system may be easily + linkable using traffic analysis. In a typical situation, where you're only reachable via Tor, this should suffice: - ./pivxd -proxy=127.0.0.1:9050 -externalip=pivxzj6l4cvo2fxy.onion -listen + ./pivxd -proxy=127.0.0.1:9050 -externalip=7zvj7a2imdgkdbg4f2dryd5rgtrn7upivr5eeij4cicjh65pooxeshid.onion -listen (obviously, replace the .onion address with your own). It should be noted that you still listen on all devices and another node could establish a clearnet connection, when knowing your address. To mitigate this, additionally bind the address of your Tor proxy: - ./pivxd ... -bind=127.0.0.1 + ./pivxd ... -bind=127.0.0.1 If you don't care too much about hiding your node, and want to be reachable on IPv4 as well, use `discover` instead: - ./pivxd ... -discover + ./pivxd ... -discover and open port 51472 on your firewall (or use port mapping, i.e., `-upnp` or `-natpmp`). If you only want to use Tor to reach .onion addresses, but not use it as a proxy for normal IPv4/IPv6 communication, use: - ./pivxd -onion=127.0.0.1:9050 -externalip=pivxzj6l4cvo2fxy.onion -discover - -## 3. Automatically listen on Tor - -Starting with Tor version 0.2.7.1 it is possible, through Tor's control socket -API, to create and destroy 'ephemeral' hidden services programmatically. -PIVX Core has been updated to make use of this. - -This means that if Tor is running (and proper authentication has been configured), -PIVX Core automatically creates a hidden service to listen on. This will positively -affect the number of available .onion nodes. - -This new feature is enabled by default if PIVX Core is listening (`-listen`), and -requires a Tor connection to work. It can be explicitly disabled with `-listenonion=0` -and, if not disabled, configured using the `-torcontrol` and `-torpassword` settings. -To show verbose debugging information, pass `-debug=tor`. - -Connecting to Tor's control socket API requires one of two authentication methods to be -configured. It also requires the control socket to be enabled, e.g. put `ControlPort 9051` -in `torrc` config file. For cookie authentication the user running pivxd must have read -access to the `CookieAuthFile` specified in Tor configuration. In some cases this is -preconfigured and the creation of a hidden service is automatic. If permission problems -are seen with `-debug=tor` they can be resolved by adding both the user running Tor and -the user running pivxd to the same group and setting permissions appropriately. On -Debian-based systems the user running pivxd can be added to the debian-tor group, -which has the appropriate permissions. - -An alternative authentication method is the use -of the `-torpassword=password` option. The `password` is the clear text form that -was used when generating the hashed password for the `HashedControlPassword` option -in the tor configuration file. The hashed password can be obtained with the command -`tor --hash-password password` (read the tor manual for more details). + ./pivxd -onion=127.0.0.1:9050 -externalip=7zvj7a2imdgkdbg4f2dryd5rgtrn7upivr5eeij4cicjh65pooxeshid.onion -discover ## 4. Privacy recommendations -- Do not add anything but PIVX Core ports to the hidden service created in section 2. - If you run a web service too, create a new hidden service for that. - Otherwise it is trivial to link them, which may reduce privacy. Hidden - services created automatically (as in section 3) always have only one port +- Do not add anything but PIVX Core ports to the onion service created in section 3. + If you run a web service too, create a new onion service for that. + Otherwise it is trivial to link them, which may reduce privacy. Onion + services created automatically (as in section 2) always have only one port open. From 2cde8e0b0f4e42e7d8c71c2cc78304350ed563a9 Mon Sep 17 00:00:00 2001 From: furszy Date: Thu, 29 Jul 2021 20:57:41 -0300 Subject: [PATCH 51/58] GUI: Do not show the tor v3 onion address in the topbar. v2 addresses were 16 chars long, new v3 addr is 56 chars long and doesn't fit on the topbar. A future good work will be to create a tor-only information dialog, grouping and presenting the tor related information properly. --- src/qt/pivx/topbar.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/qt/pivx/topbar.cpp b/src/qt/pivx/topbar.cpp index 0b4ace9990f74..5fb7744d3897d 100644 --- a/src/qt/pivx/topbar.cpp +++ b/src/qt/pivx/topbar.cpp @@ -608,8 +608,8 @@ void TopBar::updateTorIcon() ui->pushButtonTor->setChecked(true); ui->pushButtonTor->setButtonClassStyle("cssClass", "btn-check-tor", true); } - QString ip_port_q = QString::fromStdString(ip_port); - ui->pushButtonTor->setButtonText(tr("Tor Active: %1").arg(ip_port_q)); + ui->pushButtonTor->setButtonText(tr("Tor Active")); + ui->pushButtonTor->setToolTip("Address: " + QString::fromStdString(ip_port)); } else { if (ui->pushButtonTor->isChecked()) { ui->pushButtonTor->setChecked(false); From 337d43dad1e02c4b90b364f580394e833dd18fe4 Mon Sep 17 00:00:00 2001 From: Vasil Dimov Date: Mon, 12 Oct 2020 10:25:08 +0200 Subject: [PATCH 52/58] tests: don't export in6addr_loopback Don't export `in6addr_loopback` because that upsets `contrib/devtools/symbol-check.py` Fixes https://github.com/bitcoin/bitcoin/issues/20127 --- src/test/netbase_tests.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 6f74b5ddd6b56..218ca1081f2fe 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -395,17 +395,17 @@ BOOST_AUTO_TEST_CASE(netbase_dont_resolve_strings_with_embedded_nul_characters) static const std::vector fixture_addresses({ CAddress( - CService(CNetAddr(in6addr_loopback), 0 /* port */), + CService(CNetAddr(in6_addr(IN6ADDR_LOOPBACK_INIT)), 0 /* port */), NODE_NONE, 0x4966bc61U /* Fri Jan 9 02:54:25 UTC 2009 */ ), CAddress( - CService(CNetAddr(in6addr_loopback), 0x00f1 /* port */), + CService(CNetAddr(in6_addr(IN6ADDR_LOOPBACK_INIT)), 0x00f1 /* port */), NODE_NETWORK, 0x83766279U /* Tue Nov 22 11:22:33 UTC 2039 */ ), CAddress( - CService(CNetAddr(in6addr_loopback), 0xf1f2 /* port */), + CService(CNetAddr(in6_addr(IN6ADDR_LOOPBACK_INIT)), 0xf1f2 /* port */), static_cast(NODE_BLOOM), 0xffffffffU /* Sun Feb 7 06:28:15 UTC 2106 */ ) From b4515dc68413461d4560b5e3420291eb7b3c8014 Mon Sep 17 00:00:00 2001 From: furszy Date: Sun, 8 Aug 2021 13:40:34 -0300 Subject: [PATCH 53/58] GUI: Present v3 onion addresses properly in MNs list. And correct MN start not working if the wallet is running both sides. --- src/masternode.h | 1 + src/qt/pivx/masternodeswidget.cpp | 3 +++ src/qt/pivx/mnrow.cpp | 3 ++- src/qt/pivx/mnrow.h | 2 +- 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/masternode.h b/src/masternode.h index 0e717a10ae384..6db193a89f2ec 100644 --- a/src/masternode.h +++ b/src/masternode.h @@ -113,6 +113,7 @@ class CMasternode : public CSignedMessage uint256 GetSignatureHash() const override; std::string GetStrMessage() const override; const CTxIn GetVin() const { return vin; }; + CPubKey GetPubKey() const { return pubKeyMasternode; } void SetLastPing(const CMasternodePing& _lastPing) { WITH_LOCK(cs, lastPing = _lastPing;); } diff --git a/src/qt/pivx/masternodeswidget.cpp b/src/qt/pivx/masternodeswidget.cpp index 53fa9ce1d8ba7..20d8d3b30099a 100644 --- a/src/qt/pivx/masternodeswidget.cpp +++ b/src/qt/pivx/masternodeswidget.cpp @@ -248,6 +248,9 @@ bool MasterNodesWidget::startMN(const CMasternodeConfig::CMasternodeEntry& mne, return false; mnodeman.UpdateMasternodeList(mnb); + if (activeMasternode.pubKeyMasternode == mnb.GetPubKey()) { + activeMasternode.EnableHotColdMasterNode(mnb.vin, mnb.addr); + } mnb.Relay(); return true; } diff --git a/src/qt/pivx/mnrow.cpp b/src/qt/pivx/mnrow.cpp index 603a44431ebd1..d24982e3a4e5f 100644 --- a/src/qt/pivx/mnrow.cpp +++ b/src/qt/pivx/mnrow.cpp @@ -17,9 +17,10 @@ MNRow::MNRow(QWidget *parent) : ui->lblDivisory->setStyleSheet("background-color:#bababa;"); } -void MNRow::updateView(QString address, QString label, QString status, bool wasCollateralAccepted) +void MNRow::updateView(QString address, const QString& label, QString status, bool wasCollateralAccepted) { ui->labelName->setText(label); + address = address.size() < 40 ? address : address.left(20) + "..." + address.right(20); ui->labelAddress->setText(address); if (!wasCollateralAccepted) status = tr("Collateral tx not found"); ui->labelDate->setText(tr("Status: %1").arg(status)); diff --git a/src/qt/pivx/mnrow.h b/src/qt/pivx/mnrow.h index 7f596c877899e..eac8aa514908b 100644 --- a/src/qt/pivx/mnrow.h +++ b/src/qt/pivx/mnrow.h @@ -19,7 +19,7 @@ class MNRow : public QWidget explicit MNRow(QWidget *parent = nullptr); ~MNRow(); - void updateView(QString address, QString label, QString status, bool wasCollateralAccepted); + void updateView(QString address, const QString& label, QString status, bool wasCollateralAccepted); Q_SIGNALS: void onMenuClicked(); From 34ff7a87fabb9948fd5dc119f02888fc12b2bb2e Mon Sep 17 00:00:00 2001 From: furszy Date: Sun, 8 Aug 2021 19:31:40 -0300 Subject: [PATCH 54/58] Consensus: Add mnb ADDRv2 guard. So upgraded peers don't create, accept nor broadcast an mnb containing a new addr v2 to non-upgraded peers before the new NU enforcement (introduced in #2492). --- src/masternode.cpp | 10 +++++++++- src/masternode.h | 2 +- src/masternodeman.cpp | 13 +++++++++++-- src/qt/pivx/masternodeswidget.cpp | 2 +- src/rpc/masternode.cpp | 2 +- 5 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/masternode.cpp b/src/masternode.cpp index 4b244326c713a..4899d2e8e264d 100644 --- a/src/masternode.cpp +++ b/src/masternode.cpp @@ -253,7 +253,8 @@ bool CMasternodeBroadcast::Create(const std::string& strService, const std::string& strOutputIndex, std::string& strErrorRet, CMasternodeBroadcast& mnbRet, - bool fOffline) + bool fOffline, + int chainHeight) { CTxIn txin; CPubKey pubKeyCollateralAddressNew; @@ -293,6 +294,13 @@ bool CMasternodeBroadcast::Create(const std::string& strService, if (!CheckDefaultPort(_service, strErrorRet, "CMasternodeBroadcast::Create")) return false; + // Check if the MN has a ADDRv2 and reject it if the new NU wasn't enforced. + if (!_service.IsAddrV1Compatible() && + !Params().GetConsensus().NetworkUpgradeActive(chainHeight, Consensus::UPGRADE_V5_3)) { + strErrorRet = "Cannot start MN with a v2 address before the v5.3 enforcement\n"; + return false; + } + return Create(txin, _service, keyCollateralAddressNew, pubKeyCollateralAddressNew, keyMasternodeNew, pubKeyMasternodeNew, strErrorRet, mnbRet); } diff --git a/src/masternode.h b/src/masternode.h index 6db193a89f2ec..5257104768bc4 100644 --- a/src/masternode.h +++ b/src/masternode.h @@ -271,7 +271,7 @@ class CMasternodeBroadcast : public CMasternode /// Create Masternode broadcast, needs to be relayed manually after that static bool Create(const CTxIn& vin, const CService& service, const CKey& keyCollateralAddressNew, const CPubKey& pubKeyCollateralAddressNew, const CKey& keyMasternodeNew, const CPubKey& pubKeyMasternodeNew, std::string& strErrorRet, CMasternodeBroadcast& mnbRet); - static bool Create(const std::string& strService, const std::string& strKey, const std::string& strTxHash, const std::string& strOutputIndex, std::string& strErrorRet, CMasternodeBroadcast& mnbRet, bool fOffline = false); + static bool Create(const std::string& strService, const std::string& strKey, const std::string& strTxHash, const std::string& strOutputIndex, std::string& strErrorRet, CMasternodeBroadcast& mnbRet, bool fOffline, int chainHeight); static bool CheckDefaultPort(CService service, std::string& strErrorRet, const std::string& strContext); }; diff --git a/src/masternodeman.cpp b/src/masternodeman.cpp index 8fd10a466eb33..d71c94372d19a 100644 --- a/src/masternodeman.cpp +++ b/src/masternodeman.cpp @@ -712,6 +712,15 @@ int CMasternodeMan::ProcessMNBroadcast(CNode* pfrom, CMasternodeBroadcast& mnb) return 0; } + int chainHeight = GetBestHeight(); + const auto& consensus = Params().GetConsensus(); + // Check if mnb contains a ADDRv2 and reject it if the new NU wasn't enforced. + if (!mnb.addr.IsAddrV1Compatible() && + !consensus.NetworkUpgradeActive(chainHeight, Consensus::UPGRADE_V5_3)) { + LogPrint(BCLog::MASTERNODE, "mnb - received a ADDRv2 before enforcement\n"); + return 33; + } + int nDoS = 0; if (!mnb.CheckAndUpdate(nDoS, GetBestHeight())) { return nDoS; @@ -720,7 +729,7 @@ int CMasternodeMan::ProcessMNBroadcast(CNode* pfrom, CMasternodeBroadcast& mnb) // make sure the vout that was signed is related to the transaction that spawned the Masternode // - this is expensive, so it's only done once per Masternode if (!mnb.IsInputAssociatedWithPubkey()) { - LogPrintf("CMasternodeMan::ProcessMessage() : mnb - Got mismatched pubkey and vin\n"); + LogPrint(BCLog::MASTERNODE, "%s : mnb - Got mismatched pubkey and vin\n", __func__); return 33; } @@ -729,7 +738,7 @@ int CMasternodeMan::ProcessMNBroadcast(CNode* pfrom, CMasternodeBroadcast& mnb) // make sure it's still unspent // - this is checked later by .check() in many places and by ThreadCheckObfuScationPool() - if (mnb.CheckInputsAndAdd(GetBestHeight(), nDoS)) { + if (mnb.CheckInputsAndAdd(chainHeight, nDoS)) { // use this as a peer g_connman->AddNewAddress(CAddress(mnb.addr, NODE_NETWORK), pfrom->addr, 2 * 60 * 60); masternodeSync.AddedMasternodeList(mnbHash); diff --git a/src/qt/pivx/masternodeswidget.cpp b/src/qt/pivx/masternodeswidget.cpp index 20d8d3b30099a..b0231036dca90 100644 --- a/src/qt/pivx/masternodeswidget.cpp +++ b/src/qt/pivx/masternodeswidget.cpp @@ -244,7 +244,7 @@ void MasterNodesWidget::updateModelAndInform(QString informText) bool MasterNodesWidget::startMN(const CMasternodeConfig::CMasternodeEntry& mne, std::string& strError) { CMasternodeBroadcast mnb; - if (!CMasternodeBroadcast::Create(mne.getIp(), mne.getPrivKey(), mne.getTxHash(), mne.getOutputIndex(), strError, mnb)) + if (!CMasternodeBroadcast::Create(mne.getIp(), mne.getPrivKey(), mne.getTxHash(), mne.getOutputIndex(), strError, mnb, false, walletModel->getLastBlockProcessedNum())) return false; mnodeman.UpdateMasternodeList(mnb); diff --git a/src/rpc/masternode.cpp b/src/rpc/masternode.cpp index c4a6604db726c..7c2fa4a64260e 100644 --- a/src/rpc/masternode.cpp +++ b/src/rpc/masternode.cpp @@ -345,7 +345,7 @@ bool StartMasternodeEntry(UniValue& statusObjRet, CMasternodeBroadcast& mnbRet, if (strCommand == "disabled" && pmn->IsEnabled()) return false; } - fSuccessRet = CMasternodeBroadcast::Create(mne.getIp(), mne.getPrivKey(), mne.getTxHash(), mne.getOutputIndex(), errorMessage, mnbRet); + fSuccessRet = CMasternodeBroadcast::Create(mne.getIp(), mne.getPrivKey(), mne.getTxHash(), mne.getOutputIndex(), errorMessage, mnbRet, false, mnodeman.GetBestHeight()); statusObjRet.pushKV("alias", mne.getAlias()); statusObjRet.pushKV("result", fSuccessRet ? "success" : "failed"); From 015298cd236b78afe030978c4ff0d700e2a6cd78 Mon Sep 17 00:00:00 2001 From: furszy Date: Mon, 9 Aug 2021 15:25:40 -0300 Subject: [PATCH 55/58] fix: tor: Call event_base_loopbreak from the event's callback --- src/torcontrol.cpp | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 4f07dbef4ad93..607c4672640df 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -78,7 +78,7 @@ class TorControlConnection /** Create a new TorControlConnection. */ - TorControlConnection(struct event_base *base); + explicit TorControlConnection(struct event_base *base); ~TorControlConnection(); /** @@ -93,7 +93,7 @@ class TorControlConnection /** * Disconnect from Tor control port. */ - bool Disconnect(); + void Disconnect(); /** Send a command, register a handler for the reply. * A trailing CRLF is automatically added. @@ -123,7 +123,7 @@ class TorControlConnection }; TorControlConnection::TorControlConnection(struct event_base *_base): - base(_base), b_conn(0) + base(_base), b_conn(nullptr) { } @@ -140,8 +140,8 @@ void TorControlConnection::readcb(struct bufferevent *bev, void *ctx) size_t n_read_out = 0; char *line; assert(input); - // If there is not a whole line to read, evbuffer_readln returns NULL - while((line = evbuffer_readln(input, &n_read_out, EVBUFFER_EOL_CRLF)) != NULL) + // If there is not a whole line to read, evbuffer_readln returns nullptr + while((line = evbuffer_readln(input, &n_read_out, EVBUFFER_EOL_CRLF)) != nullptr) { std::string s(line, n_read_out); free(line); @@ -212,7 +212,7 @@ bool TorControlConnection::Connect(const std::string &target, const ConnectionCB b_conn = bufferevent_socket_new(base, -1, BEV_OPT_CLOSE_ON_FREE); if (!b_conn) return false; - bufferevent_setcb(b_conn, TorControlConnection::readcb, NULL, TorControlConnection::eventcb, this); + bufferevent_setcb(b_conn, TorControlConnection::readcb, nullptr, TorControlConnection::eventcb, this); bufferevent_enable(b_conn, EV_READ|EV_WRITE); this->connected = _connected; this->disconnected = _disconnected; @@ -225,12 +225,11 @@ bool TorControlConnection::Connect(const std::string &target, const ConnectionCB return true; } -bool TorControlConnection::Disconnect() +void TorControlConnection::Disconnect() { if (b_conn) bufferevent_free(b_conn); - b_conn = 0; - return true; + b_conn = nullptr; } bool TorControlConnection::Command(const std::string &cmd, const ReplyHandlerCB& reply_handler) @@ -335,7 +334,7 @@ static std::map ParseTorReplyMapping(const std::string if (j == 3 && value[i] > '3') { j--; } - escaped_value.push_back(strtol(value.substr(i, j).c_str(), NULL, 8)); + escaped_value.push_back(strtol(value.substr(i, j).c_str(), nullptr, 8)); // Account for automatic incrementing at loop end i += j - 1; } else { @@ -369,7 +368,7 @@ static std::map ParseTorReplyMapping(const std::string static std::pair ReadBinaryFile(const fs::path &filename, size_t maxsize=std::numeric_limits::max()) { FILE *f = fsbridge::fopen(filename, "rb"); - if (f == NULL) + if (f == nullptr) return std::make_pair(false,""); std::string retval; char buffer[128]; @@ -395,7 +394,7 @@ static std::pair ReadBinaryFile(const fs::path &filename, size static bool WriteBinaryFile(const fs::path &filename, const std::string &data) { FILE *f = fsbridge::fopen(filename, "wb"); - if (f == NULL) + if (f == nullptr) return false; if (fwrite(data.data(), 1, data.size(), f) != data.size()) { fclose(f); @@ -408,7 +407,7 @@ static bool WriteBinaryFile(const fs::path &filename, const std::string &data) /****** Bitcoin specific TorController implementation ********/ /** Controller that connects to Tor control socket, authenticate, then create - * and maintain a ephemeral hidden service. + * and maintain an ephemeral onion service. */ class TorController { @@ -416,7 +415,7 @@ class TorController TorController(struct event_base* base, const std::string& target); ~TorController(); - /** Get name fo file to store private key in */ + /** Get name of file to store private key in */ fs::path GetPrivateKeyFile(); /** Reconnect, after getting disconnected */ @@ -478,7 +477,7 @@ TorController::~TorController() { if (reconnect_ev) { event_free(reconnect_ev); - reconnect_ev = 0; + reconnect_ev = nullptr; } if (service.IsValid()) { RemoveLocal(service); @@ -763,7 +762,9 @@ void InterruptTorControl() { if (gBase) { LogPrintf("tor: Thread interrupt\n"); - event_base_loopbreak(gBase); + event_base_once(gBase, -1, EV_TIMEOUT, [](evutil_socket_t, short, void*) { + event_base_loopbreak(gBase); + }, nullptr, nullptr); } } From f44be9471bd2d8dde7d251e979317b06d0c07da2 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 11 Oct 2020 00:13:10 -0700 Subject: [PATCH 56/58] Only relay IPv4, IPv6, Tor addresses --- src/net_processing.cpp | 1 + src/netaddress.h | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 0c3911e6fbbb7..73566e18e28d5 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -822,6 +822,7 @@ static void RelayTransaction(const CTransaction& tx, CConnman* connman) static void RelayAddress(const CAddress& addr, bool fReachable, CConnman* connman) { + if (!fReachable && !addr.IsRelayable()) return; int nRelayNodes = fReachable ? 2 : 1; // limited relaying of addresses outside our network(s) // Relay to a limited number of other nodes diff --git a/src/netaddress.h b/src/netaddress.h index 7c0c19a5e5fbe..7c5fbd26166ea 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -211,6 +211,14 @@ class CNetAddr friend bool operator!=(const CNetAddr& a, const CNetAddr& b); friend bool operator<(const CNetAddr& a, const CNetAddr& b); + /** + * Whether this address should be relayed to other peers even if we can't reach it ourselves. + */ + bool IsRelayable() const + { + return IsIPv4() || IsIPv6() || IsTor(); + } + /** * Serialize to a stream. */ From b63e4f51a73a43d1f9bd43a1d637e197c368c6d8 Mon Sep 17 00:00:00 2001 From: furszy Date: Tue, 10 Aug 2021 09:51:30 -0300 Subject: [PATCH 57/58] Consensus: Add v5.3 enforcement height for testnet. --- src/chainparams.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/chainparams.cpp b/src/chainparams.cpp index ba5672c84cee4..1f998f17aadbe 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -342,8 +342,7 @@ class CTestNetParams : public CChainParams consensus.vUpgrades[Consensus::UPGRADE_V4_0].nActivationHeight = 201; consensus.vUpgrades[Consensus::UPGRADE_V5_0].nActivationHeight = 201; consensus.vUpgrades[Consensus::UPGRADE_V5_2].nActivationHeight = 262525; - consensus.vUpgrades[Consensus::UPGRADE_V5_3].nActivationHeight = - Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT; + consensus.vUpgrades[Consensus::UPGRADE_V5_3].nActivationHeight = 332300; consensus.vUpgrades[Consensus::UPGRADE_V6_0].nActivationHeight = Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT; From ecde04af3455e1b803ae93bea059f72915a1765d Mon Sep 17 00:00:00 2001 From: random-zebra Date: Tue, 10 Aug 2021 15:20:09 +0200 Subject: [PATCH 58/58] [Consensus] Bump Active Protocol version to 70923 for v5.3 --- src/validation.cpp | 12 ++++++------ src/version.h | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/validation.cpp b/src/validation.cpp index 9e662f5e119df..2ef74a4e6817c 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -4129,14 +4129,14 @@ void static CheckBlockIndex() // it was the one which was commented out int ActiveProtocol() { - // SPORK_14 is used for 70922 (v5.2.0) - if (sporkManager.IsSporkActive(SPORK_14_NEW_PROTOCOL_ENFORCEMENT)) - return MIN_PEER_PROTO_VERSION_AFTER_ENFORCEMENT; - - // SPORK_15 was used for 70921 (v5.0.1), commented out now. - //if (sporkManager.IsSporkActive(SPORK_15_NEW_PROTOCOL_ENFORCEMENT_2)) + // SPORK_14 was used for 70922 (v5.2.0), commented out now. + //if (sporkManager.IsSporkActive(SPORK_14_NEW_PROTOCOL_ENFORCEMENT)) // return MIN_PEER_PROTO_VERSION_AFTER_ENFORCEMENT; + // SPORK_15 is used for 70923 (v5.3.0) + if (sporkManager.IsSporkActive(SPORK_15_NEW_PROTOCOL_ENFORCEMENT_2)) + return MIN_PEER_PROTO_VERSION_AFTER_ENFORCEMENT; + return MIN_PEER_PROTO_VERSION_BEFORE_ENFORCEMENT; } diff --git a/src/version.h b/src/version.h index 9df15fe40c0df..506b4ebb9fe52 100644 --- a/src/version.h +++ b/src/version.h @@ -20,8 +20,8 @@ static const int INIT_PROTO_VERSION = 209; static const int GETHEADERS_VERSION = 70077; //! disconnect from peers older than this proto version -static const int MIN_PEER_PROTO_VERSION_BEFORE_ENFORCEMENT = 70921; -static const int MIN_PEER_PROTO_VERSION_AFTER_ENFORCEMENT = 70922; +static const int MIN_PEER_PROTO_VERSION_BEFORE_ENFORCEMENT = 70922; +static const int MIN_PEER_PROTO_VERSION_AFTER_ENFORCEMENT = 70923; //! peers with version older than this, could relay invalid (stale) mn pings static const int MIN_PEER_CACHEDVERSION = 70921;