Merge #2480: [Net] asmap to improve IP bucketing in addrman - backports

16791f2 CMakeLists tests: add raw files generation. (furszy)
672d9a2 init: move asmap code earlier in init process (Jon Atack)
65cd143 net: extract conditional to bool CNetAddr::IsHeNet (Jon Atack)
2fc1f37 logging: asmap logging and #include fixups (Jon Atack)
0c9efb8 test: add functional test for an empty, unparsable asmap (Jon Atack)
6545656 config: separate the asmap finding and parsing checks (Jon Atack)
618b8d1 config: enable passing -asmap an absolute file path (Jon Atack)
8c7bdbe config: use default value in -asmap config (Jon Atack)
de39fab test: add feature_asmap functional tests (Jon Atack)
4290d3f Make asmap Interpret tolerant of malicious map data (Pieter Wuille)
e527e04 Use ASNs for mapped IPv4 addresses correctly (Pieter Wuille)
9a28bc0 Mark asmap const in statistics code (Pieter Wuille)
868a6ed Avoid asmap copies in initialization (Pieter Wuille)
cb698fb Add extra logging of asmap use and bucketing (Gleb Naumenko)
2fe5a05 Return mapped AS in RPC call getpeerinfo (Gleb Naumenko)
ce7aa15 scripted-diff: Replace NET_TOR with NET_ONION (wodry)
4c3ae7d Integrate ASN bucketing in Addrman and add tests (Gleb Naumenko)
718f1df CAddrManTest: remove redundant MakeDeterministic call. (furszy)
fd51941 Tests: address placement should be deterministic by default (René Nyffenegger)
8d01cbd  Add asmap utility which queries a mapping (Gleb Naumenko)
e986ed0 CAddrMan::Deserialize handle corrupt serializations better. (Patrick Strateman)
d2a8baf addrman.h: CAddrInfo inline members default values, plus several typos corrected. (furszy)
a7b9fd9 refactor: Use uint16_t instead of unsigned short (furszy)

Pull request description:

  Decoupled from #2411, built on top of #2479. Probably the last decouple from the "road to Tor" work.

  Focused on porting the ASN nodes bucketing functionality. The hearth of this work is bitcoin#16702.

  Providing an asmap file that contains the IP->ASN mapping, nodes will be bucketed by AS they belong to, in order to make impossible for a node to connect to several nodes hosted in a single AS.
  This is done in response to Erebus attack, but also to generally diversify the connections every node creates, especially useful when a large fraction of nodes operate under a couple of cloud providers.

  #### List of PRs:
  * bitcoin#7932
  * bitcoin#10765
  * bitcoin#13532
  * bitcoin#13575
  * bitcoin#16702
  * bitcoin#17812
  * bitcoin#18023
  * bitcoin#19314

  PRs for a follow up PR:
  * bitcoin#18029
  * bitcoin#18512

ACKs for top commit:
  random-zebra:
    re-utACK 16791f2
  Fuzzbawls:
    ACK 16791f2

Tree-SHA512: 1452af87d693526d3359822845bbd6211578b5c7c69d740d19c8c3ee25c66fd6e130f4421066a8f5384d62f65a2754423c633f90d7e3d809f4f1cc00c3c956ba

CMakeLists.txt

@@ -427,6 +427,7 @@ set(UTIL_SOURCES
         ./src/threadinterrupt.cpp
         ./src/arith_uint256.cpp
         ./src/uint256.cpp
+        ./src/util/asmap.cpp
         ./src/util/threadnames.cpp
         ./src/util/blockstatecatcher.h
         ./src/util/system.cpp

src/Makefile.am

@@ -293,6 +293,7 @@ BITCOIN_CORE_H = \
   guiinterfaceutil.h \
   uint256.h \
   undo.h \
+  util/asmap.h \
   util/blockstatecatcher.h \
   util/system.h \
   util/macros.h \
@@ -561,6 +562,7 @@ libbitcoin_util_a_SOURCES = \
   support/lockedpool.cpp \
   sync.cpp \
   threadinterrupt.cpp \
+  util/asmap.cpp \
   uint256.cpp \
   util/system.cpp \
   utilmoneystr.cpp \

src/Makefile.test.include

@@ -60,7 +60,8 @@ JSON_TEST_FILES = \
   test/data/merkle_commitments_sapling.json \
   test/data/sapling_key_components.json
 
-RAW_TEST_FILES =
+RAW_TEST_FILES = \
+  test/data/asmap.raw
 
 GENERATED_TEST_FILES = $(JSON_TEST_FILES:.json=.json.h) $(RAW_TEST_FILES:.raw=.raw.h)
 
@@ -423,3 +424,12 @@ check-local: check-sapling check-standard
 	 echo "};};"; \
 	} > "$@.new" && mv -f "$@.new" "$@"
 	@echo "Generated $@"
+
+%.raw.h: %.raw
+	@$(MKDIR_P) $(@D)
+	@{ \
+	 echo "static unsigned const char $(*F)_raw[] = {" && \
+	 $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x  ,//g' && \
+	 echo "};"; \
+	} > "$@.new" && mv -f "$@.new" "$@"
+	@echo "Generated $@"

src/addrdb.cpp

@@ -15,6 +15,8 @@
 #include "tinyformat.h"
 #include "util/system.h"
 
+#include <cstdint>
+
 namespace {
 
 template <typename Stream, typename Data>
@@ -37,7 +39,7 @@ template <typename Data>
 bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data& data)
 {
     // Generate random temporary filename
-    unsigned short randv = 0;
+    uint16_t randv = 0;
     GetRandBytes((unsigned char*)&randv, sizeof(randv));
     std::string tmpfn = strprintf("%s.%04x", prefix, randv);
 

src/addrman.cpp

@@ -7,23 +7,30 @@
 #include "addrman.h"
 
 #include "hash.h"
-#include "serialize.h"
 #include "streams.h"
+#include "logging.h"
+#include "serialize.h"
 
 
-int CAddrInfo::GetTriedBucket(const uint256& nKey) const
+int CAddrInfo::GetTriedBucket(const uint256& nKey, const std::vector<bool> &asmap) const
 {
     uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetKey()).GetHash().GetCheapHash();
-    uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup() << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetHash().GetCheapHash();
-    return hash2 % ADDRMAN_TRIED_BUCKET_COUNT;
+    uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetHash().GetCheapHash();
+    int tried_bucket = hash2 % ADDRMAN_TRIED_BUCKET_COUNT;
+    uint32_t mapped_as = GetMappedAS(asmap);
+    LogPrint(BCLog::NET, "IP %s mapped to AS%i belongs to tried bucket %i\n", ToStringIP(), mapped_as, tried_bucket);
+    return tried_bucket;
 }
 
-int CAddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src) const
+int CAddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const std::vector<bool> &asmap) const
 {
-    std::vector<unsigned char> vchSourceGroupKey = src.GetGroup();
-    uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup() << vchSourceGroupKey).GetHash().GetCheapHash();
+    std::vector<unsigned char> vchSourceGroupKey = src.GetGroup(asmap);
+    uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << vchSourceGroupKey).GetHash().GetCheapHash();
     uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << vchSourceGroupKey << (hash1 % ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP)).GetHash().GetCheapHash();
-    return hash2 % ADDRMAN_NEW_BUCKET_COUNT;
+    int new_bucket = hash2 % ADDRMAN_NEW_BUCKET_COUNT;
+    uint32_t mapped_as = GetMappedAS(asmap);
+    LogPrint(BCLog::NET, "IP %s mapped to AS%i belongs to new bucket %i\n", ToStringIP(), mapped_as, new_bucket);
+    return new_bucket;
 }
 
 int CAddrInfo::GetBucketPosition(const uint256& nKey, bool fNew, int nBucket) const
@@ -156,7 +163,7 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId)
     assert(info.nRefCount == 0);
 
     // which tried bucket to move the entry to
-    int nKBucket = info.GetTriedBucket(nKey);
+    int nKBucket = info.GetTriedBucket(nKey, m_asmap);
     int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket);
 
     // first make space to add it (the existing tried entry there is moved to new, deleting whatever is there).
@@ -172,7 +179,7 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId)
         nTried--;
 
         // find which new bucket it belongs to
-        int nUBucket = infoOld.GetNewBucket(nKey);
+        int nUBucket = infoOld.GetNewBucket(nKey, m_asmap);
         int nUBucketPos = infoOld.GetBucketPosition(nKey, true, nUBucket);
         ClearNew(nUBucket, nUBucketPos);
         assert(vvNew[nUBucket][nUBucketPos] == -1);
@@ -236,7 +243,7 @@ void CAddrMan::Good_(const CService& addr, bool test_before_evict, int64_t nTime
         return;
 
     // which tried bucket to move the entry to
-    int tried_bucket = info.GetTriedBucket(nKey);
+    int tried_bucket = info.GetTriedBucket(nKey, m_asmap);
     int tried_bucket_pos = info.GetBucketPosition(nKey, false, tried_bucket);
 
     // Will moving this address into tried evict another entry?
@@ -304,7 +311,7 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP
         fNew = true;
     }
 
-    int nUBucket = pinfo->GetNewBucket(nKey, source);
+    int nUBucket = pinfo->GetNewBucket(nKey, source, m_asmap);
     int nUBucketPos = pinfo->GetBucketPosition(nKey, true, nUBucket);
     if (vvNew[nUBucket][nUBucketPos] != nId) {
         bool fInsert = vvNew[nUBucket][nUBucketPos] == -1;
@@ -438,15 +445,15 @@ int CAddrMan::Check_()
 
     for (int n = 0; n < ADDRMAN_TRIED_BUCKET_COUNT; n++) {
         for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) {
-            if (vvTried[n][i] != -1) {
-                if (!setTried.count(vvTried[n][i]))
-                    return -11;
-                if (mapInfo[vvTried[n][i]].GetTriedBucket(nKey) != n)
-                    return -17;
-                if (mapInfo[vvTried[n][i]].GetBucketPosition(nKey, false, n) != i)
-                    return -18;
-                setTried.erase(vvTried[n][i]);
-            }
+             if (vvTried[n][i] != -1) {
+                 if (!setTried.count(vvTried[n][i]))
+                     return -11;
+                 if (mapInfo[vvTried[n][i]].GetTriedBucket(nKey, m_asmap) != n)
+                     return -17;
+                 if (mapInfo[vvTried[n][i]].GetBucketPosition(nKey, false, n) != i)
+                     return -18;
+                 setTried.erase(vvTried[n][i]);
+             }
         }
     }
 
@@ -547,7 +554,7 @@ void CAddrMan::ResolveCollisions_()
             CAddrInfo& info_new = mapInfo[id_new];
 
             // Which tried bucket to move the entry to.
-            int tried_bucket = info_new.GetTriedBucket(nKey);
+            int tried_bucket = info_new.GetTriedBucket(nKey, m_asmap);
             int tried_bucket_pos = info_new.GetBucketPosition(nKey, false, tried_bucket);
             if (!info_new.IsValid()) { // id_new may no longer map to a valid address
                 erase_collision = true;
@@ -611,10 +618,33 @@ CAddrInfo CAddrMan::SelectTriedCollision_()
     CAddrInfo& newInfo = mapInfo[id_new];
 
     // which tried bucket to move the entry to
-    int tried_bucket = newInfo.GetTriedBucket(nKey);
+    int tried_bucket = newInfo.GetTriedBucket(nKey, m_asmap);
     int tried_bucket_pos = newInfo.GetBucketPosition(nKey, false, tried_bucket);
 
     int id_old = vvTried[tried_bucket][tried_bucket_pos];
 
     return mapInfo[id_old];
 }
+
+std::vector<bool> CAddrMan::DecodeAsmap(fs::path path)
+{
+    std::vector<bool> bits;
+    FILE *filestr = fsbridge::fopen(path, "rb");
+    CAutoFile file(filestr, SER_DISK, CLIENT_VERSION);
+    if (file.IsNull()) {
+        LogPrintf("Failed to open asmap file from disk\n");
+        return bits;
+    }
+    fseek(filestr, 0, SEEK_END);
+    int length = ftell(filestr);
+    LogPrintf("Opened asmap file %s (%d bytes) from disk\n", path, length);
+    fseek(filestr, 0, SEEK_SET);
+    char cur_byte;
+    for (int i = 0; i < length; ++i) {
+        file >> cur_byte;
+        for (int bit = 0; bit < 8; ++bit) {
+            bits.push_back((cur_byte >> bit) & 1);
+        }
+    }
+    return bits;
+}