Skip to content

Commit

Permalink
net: CAddress & CAddrMan: (un)serialize as ADDRv2
Browse files Browse the repository at this point in the history 
Change the serialization of `CAddrMan` to serialize its addresses
in ADDRv2/BIP155 format by default. Introduce a new `CAddrMan` format
version (3).

Add support for ADDRv2 format in `CAddress` (un)serialization.

Adaptation of btc@201a4596d92d640d5eb7e76cc8d959228fa09dbb
  • Loading branch information
@furszy
furszy committed Aug 10, 2021
1 parent b8c1dda commit cec9567
Show file tree
Hide file tree
Showing 5 changed files with 169 additions and 15 deletions.
7 changes: 7 additions & 0 deletions doc/release-notes.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,13 @@ From PIVX Core 6.0 onwards, macOS versions earlier than 10.12 are no longer supp

PIVX Core should also work on most other Unix-like systems but is not frequently tested on them.

The node's known peers are persisted to disk in a file called `peers.dat`. The
format of this file has been changed in a backwards-incompatible way in order to
accommodate the storage of Tor v3 and other BIP155 addresses. This means that if
the file is modified by v5.3 or newer then older versions will not be able to
read it. Those old versions, in the event of a downgrade, will log an error
message that deserialization has failed and will continue normal operation
as if the file was missing, creating a new empty one. (#2411)

Notable Changes
==============
Expand Down
56 changes: 44 additions & 12 deletions src/addrman.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
#include "random.h"
#include "sync.h"
#include "timedata.h"
#include "tinyformat.h"
#include "util/system.h"

#include <fs.h>
Expand Down Expand Up @@ -271,6 +272,14 @@ friend class CAddrManTest;
void SetServices_(const CService& addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs);

public:
//! Serialization versions.
enum class Format : uint8_t {
V0_HISTORICAL = 0, //!< historic format, before commit e6b343d88
V1_DETERMINISTIC = 1, //!< for pre-asmap files
V2_ASMAP = 2, //!< for files including asmap version
V3_BIP155 = 3, //!< same as V2_ASMAP plus addresses are in BIP155 format
};

// Compressed IP->ASN mapping, loaded from a file when a node starts.
// Should be always empty if no file was provided.
// This mapping is then used for bucketing nodes in Addrman.
Expand All @@ -292,8 +301,8 @@ friend class CAddrManTest;


/**
* serialized format:
* * version byte (1 for pre-asmap files, 2 for files including asmap version)
* Serialized format.
* * version byte (@see `Format`)
* * 0x20 + nKey (serialized as if it were a vector, for backward compatibility)
* * nNew
* * nTried
Expand Down Expand Up @@ -321,12 +330,15 @@ friend class CAddrManTest;
* very little in common.
*/
template <typename Stream>
void Serialize(Stream& s) const
void Serialize(Stream& s_) const
{
LOCK(cs);

unsigned char nVersion = 2;
s << nVersion;
// Always serialize in the latest version (currently Format::V3_BIP155).

OverrideStream<Stream> s(&s_, s_.GetType(), s_.GetVersion() | ADDRV2_FORMAT);

s << static_cast<uint8_t>(Format::V3_BIP155);
s << ((unsigned char)32);
s << nKey;
s << nNew;
Expand Down Expand Up @@ -378,13 +390,33 @@ friend class CAddrManTest;
}

template <typename Stream>
void Unserialize(Stream& s)
void Unserialize(Stream& s_)
{
LOCK(cs);

Clear();
unsigned char nVersion;
s >> nVersion;

Format format;
s_ >> Using<CustomUintFormatter<1>>(format);

static constexpr Format maximum_supported_format = Format::V3_BIP155;
if (format > maximum_supported_format) {
throw std::ios_base::failure(strprintf(
"Unsupported format of addrman database: %u. Maximum supported is %u. "
"Continuing operation without using the saved list of peers.",
static_cast<uint8_t>(format),
static_cast<uint8_t>(maximum_supported_format)));
}

int stream_version = s_.GetVersion();
if (format >= Format::V3_BIP155) {
// Add ADDRV2_FORMAT to the version so that the CNetAddr and CAddress
// unserialize methods know that an address in addrv2 format is coming.
stream_version |= ADDRV2_FORMAT;
}

OverrideStream<Stream> s(&s_, s_.GetType(), stream_version);

unsigned char nKeySize;
s >> nKeySize;
if (nKeySize != 32) throw std::ios_base::failure("Incorrect keysize in addrman deserialization");
Expand All @@ -393,7 +425,7 @@ friend class CAddrManTest;
s >> nTried;
int nUBuckets = 0;
s >> nUBuckets;
if (nVersion != 0) {
if (format >= Format::V1_DETERMINISTIC) {
nUBuckets ^= (1 << 30);
}

Expand Down Expand Up @@ -456,21 +488,21 @@ friend class CAddrManTest;
supplied_asmap_version = SerializeHash(m_asmap);
}
uint256 serialized_asmap_version;
if (nVersion > 1) {
if (format >= Format::V2_ASMAP) {
s >> serialized_asmap_version;
}

for (int n = 0; n < nNew; n++) {
CAddrInfo &info = mapInfo[n];
int bucket = entryToBucket[n];
int nUBucketPos = info.GetBucketPosition(nKey, true, bucket);
if (nVersion == 2 && nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && vvNew[bucket][nUBucketPos] == -1 &&
if (format >= Format::V2_ASMAP && nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && vvNew[bucket][nUBucketPos] == -1 &&
info.nRefCount < ADDRMAN_NEW_BUCKETS_PER_ADDRESS && serialized_asmap_version == supplied_asmap_version) {
// Bucketing has not changed, using existing bucket positions for the new table
vvNew[bucket][nUBucketPos] = n;
info.nRefCount++;
} else {
// In case the new table data cannot be used (nVersion unknown, bucket count wrong or new asmap),
// In case the new table data cannot be used (format unknown, bucket count wrong or new asmap),
// try to give them a reference based on their primary source address.
LogPrint(BCLog::ADDRMAN, "Bucketing method was updated, re-bucketing addrman entries from disk\n");
bucket = info.GetNewBucket(nKey, m_asmap);
Expand Down
12 changes: 10 additions & 2 deletions src/protocol.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,8 @@ class CAddress : public CService

public:
CAddress() : CService{} {};
explicit CAddress(CService ipIn, ServiceFlags nServicesIn) : CService{ipIn}, nServices{nServicesIn} {};
CAddress(CService ipIn, ServiceFlags nServicesIn) : CService{ipIn}, nServices{nServicesIn} {};
CAddress(CService ipIn, ServiceFlags nServicesIn, uint32_t nTimeIn) : CService{ipIn}, nServices{nServicesIn}, nTime{nTimeIn} {};

SERIALIZE_METHODS(CAddress, obj)
{
Expand All @@ -314,7 +315,14 @@ class CAddress : public CService
(nVersion >= CADDR_TIME_VERSION && !(s.GetType() & SER_GETHASH))) {
READWRITE(obj.nTime);
}
READWRITE(Using<CustomUintFormatter<8>>(obj.nServices));
if (nVersion & ADDRV2_FORMAT) {
uint64_t services_tmp;
SER_WRITE(obj, services_tmp = obj.nServices);
READWRITE(Using<CompactSizeFormatter<false>>(services_tmp));
SER_READ(obj, obj.nServices = static_cast<ServiceFlags>(services_tmp));
} else {
READWRITE(Using<CustomUintFormatter<8>>(obj.nServices));
}
READWRITEAS(CService, obj);
}

Expand Down
1 change: 1 addition & 0 deletions src/streams.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ class OverrideStream
int GetVersion() const { return nVersion; }
int GetType() const { return nType; }
size_t size() const { return stream->size(); }
void ignore(size_t size) { return stream->ignore(size); }
};

/** Double ended buffer combining vector and stream-like interfaces.
Expand Down
108 changes: 107 additions & 1 deletion src/test/netbase_tests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,14 @@
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include "test/test_pivx.h"

#include "net.h" // validateMasternodeIP
#include "netbase.h"
#include "test/test_pivx.h"
#include "protocol.h"
#include "serialize.h"
#include "streams.h"
#include "version.h"

#include <string>

Expand Down Expand Up @@ -385,4 +390,105 @@ BOOST_AUTO_TEST_CASE(netbase_dont_resolve_strings_with_embedded_nul_characters)
BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0example.com\0", 35), ret));
}

// Since CNetAddr (un)ser is tested separately in net_tests.cpp here we only
// try a few edge cases for port, service flags and time.

static const std::vector<CAddress> fixture_addresses({
CAddress(
CService(CNetAddr(in6addr_loopback), 0 /* port */),
NODE_NONE,
0x4966bc61U /* Fri Jan 9 02:54:25 UTC 2009 */
),
CAddress(
CService(CNetAddr(in6addr_loopback), 0x00f1 /* port */),
NODE_NETWORK,
0x83766279U /* Tue Nov 22 11:22:33 UTC 2039 */
),
CAddress(
CService(CNetAddr(in6addr_loopback), 0xf1f2 /* port */),
static_cast<ServiceFlags>(NODE_BLOOM),
0xffffffffU /* Sun Feb 7 06:28:15 UTC 2106 */
)
});

// fixture_addresses should equal to this when serialized in V1 format.
// When this is unserialized from V1 format it should equal to fixture_addresses.
static constexpr const char* stream_addrv1_hex =
"03" // number of entries

"61bc6649" // time, Fri Jan 9 02:54:25 UTC 2009
"0000000000000000" // service flags, NODE_NONE
"00000000000000000000000000000001" // address, fixed 16 bytes (IPv4 embedded in IPv6)
"0000" // port

"79627683" // time, Tue Nov 22 11:22:33 UTC 2039
"0100000000000000" // service flags, NODE_NETWORK
"00000000000000000000000000000001" // address, fixed 16 bytes (IPv6)
"00f1" // port

"ffffffff" // time, Sun Feb 7 06:28:15 UTC 2106
"0400000000000000" // service flags, NODE_BLOOM
"00000000000000000000000000000001" // address, fixed 16 bytes (IPv6)
"f1f2"; // port

// fixture_addresses should equal to this when serialized in V2 format.
// When this is unserialized from V2 format it should equal to fixture_addresses.
static constexpr const char* stream_addrv2_hex =
"03" // number of entries

"61bc6649" // time, Fri Jan 9 02:54:25 UTC 2009
"00" // service flags, COMPACTSIZE(NODE_NONE)
"02" // network id, IPv6
"10" // address length, COMPACTSIZE(16)
"00000000000000000000000000000001" // address
"0000" // port

"79627683" // time, Tue Nov 22 11:22:33 UTC 2039
"01" // service flags, COMPACTSIZE(NODE_NETWORK)
"02" // network id, IPv6
"10" // address length, COMPACTSIZE(16)
"00000000000000000000000000000001" // address
"00f1" // port

"ffffffff" // time, Sun Feb 7 06:28:15 UTC 2106
"04" // service flags, COMPACTSIZE(NODE_BLOOM)
"02" // network id, IPv6
"10" // address length, COMPACTSIZE(16)
"00000000000000000000000000000001" // address
"f1f2"; // port

BOOST_AUTO_TEST_CASE(caddress_serialize_v1)
{
CDataStream s(SER_NETWORK, PROTOCOL_VERSION);

s << fixture_addresses;
BOOST_CHECK_EQUAL(HexStr(s), stream_addrv1_hex);
}

BOOST_AUTO_TEST_CASE(caddress_unserialize_v1)
{
CDataStream s(ParseHex(stream_addrv1_hex), SER_NETWORK, PROTOCOL_VERSION);
std::vector<CAddress> addresses_unserialized;

s >> addresses_unserialized;
BOOST_CHECK(fixture_addresses == addresses_unserialized);
}

BOOST_AUTO_TEST_CASE(caddress_serialize_v2)
{
CDataStream s(SER_NETWORK, PROTOCOL_VERSION | ADDRV2_FORMAT);

s << fixture_addresses;
BOOST_CHECK_EQUAL(HexStr(s), stream_addrv2_hex);
}

BOOST_AUTO_TEST_CASE(caddress_unserialize_v2)
{
CDataStream s(ParseHex(stream_addrv2_hex), SER_NETWORK, PROTOCOL_VERSION | ADDRV2_FORMAT);
std::vector<CAddress> addresses_unserialized;

s >> addresses_unserialized;
BOOST_CHECK(fixture_addresses == addresses_unserialized);
}

BOOST_AUTO_TEST_SUITE_END()

0 comments on commit cec9567

Please sign in to comment.