Network tests & updates

lib/net/hostlist.js

@@ -23,6 +23,42 @@ const NetAddress = require('./netaddress');
 const common = require('./common');
 const seeds = require('./seeds');
 
+/**
+ * Stochastic address manager based on bitcoin addrman.
+ *
+ * Design goals:
+ *  * Keep the address tables in-memory, and asynchronously dump the entire
+ *    table to hosts.json.
+ *  * Make sure no (localized) attacker can fill the entire table with his
+ *    nodes/addresses.
+ *
+ * To that end:
+ *  * Addresses are organized into buckets that can each store up
+ *    to 64 entries (maxEntries).
+ *    * Addresses to which our node has not successfully connected go into
+ *      1024 "fresh" buckets (maxFreshBuckets).
+ *      * Based on the address range of the source of information
+ *        64 buckets are selected at random.
+ *      * The actual bucket is chosen from one of these, based on the range in
+ *        which the address itself is located.
+ *      * The position in the bucket is chosen based on the full address.
+ *      * One single address can occur in up to 8 different buckets to increase
+ *        selection chances for addresses that are seen frequently. The chance
+ *        for increasing this multiplicity decreases exponentially.
+ *      * When adding a new address to an occupied position of a bucket, it
+ *        will not replace the existing entry unless that address is also stored
+ *        in another bucket or it doesn't meet one of several quality criteria
+ *        (see isStale for exact criteria).
+ *    * Addresses of nodes that are known to be accessible go into 256 "tried"
+ *      buckets.
+ *      * Each address range selects at random 8 of these buckets.
+ *      * The actual bucket is chosen from one of these, based on the full
+ *        address.
+ *    * Bucket selection is based on cryptographic hashing,
+ *      using a randomly-generated 256-bit key, which should not be observable
+ *      by adversaries (key).
+ */
+
 /**
  * Host List
  * @alias module:net.HostList
@@ -42,6 +78,7 @@ class HostList {
     this.address = this.options.address;
     this.brontide = this.options.brontide;
     this.resolve = this.options.resolve;
+    this.random = this.options.random;
 
     this.key = rng.randomBytes(32);
     this.hash = new Hash256();
@@ -381,7 +418,7 @@ class HostList {
       buckets = this.fresh;
 
     if (this.totalUsed > 0) {
-      if (this.totalFresh === 0 || random(2) === 0)
+      if (this.totalFresh === 0 || this.random(2) === 0)
         buckets = this.used;
     }
 
@@ -393,13 +430,13 @@ class HostList {
     let factor = 1;
 
     for (;;) {
-      const i = random(buckets.length);
+      const i = this.random(buckets.length);
       const bucket = buckets[i];
 
       if (bucket.size === 0)
         continue;
 
-      let index = random(bucket.size);
+      let index = this.random(bucket.size);
       let entry;
 
       if (buckets === this.used) {
@@ -414,7 +451,7 @@ class HostList {
         }
       }
 
-      const num = random(1 << 30);
+      const num = this.random(1 << 30);
 
       if (num < factor * entry.chance(now) * (1 << 30))
         return entry;
@@ -427,17 +464,20 @@ class HostList {
    * Get fresh bucket for host.
    * @private
    * @param {HostEntry} entry
+   * @param {NetAddress?} src
    * @returns {Map}
    */
 
-  freshBucket(entry) {
+  freshBucket(entry, src) {
     const addr = entry.addr;
-    const src = entry.src;
+
+    if (!src)
+      src = entry.src;
 
     this.hash.init();
     this.hash.update(this.key);
-    this.hash.update(groupKey(addr.raw));
-    this.hash.update(groupKey(src.raw));
+    this.hash.update(addr.getGroup());
+    this.hash.update(src.getGroup());
 
     const hash1 = this.hash.final();
     const hash32 = bio.readU32(hash1, 0) % 64;
@@ -446,7 +486,7 @@ class HostList {
 
     this.hash.init();
     this.hash.update(this.key);
-    this.hash.update(groupKey(src.raw));
+    this.hash.update(src.getGroup());
     this.hash.update(this.hashbuf);
 
     const hash2 = this.hash.final();
@@ -481,7 +521,7 @@ class HostList {
 
     this.hash.init();
     this.hash.update(this.key);
-    this.hash.update(groupKey(addr.raw));
+    this.hash.update(addr.getGroup());
     this.hash.update(this.hashbuf);
 
     const hash2 = this.hash.final();
@@ -552,7 +592,7 @@ class HostList {
       for (let i = 0; i < entry.refCount; i++)
         factor *= 2;
 
-      if (random(factor) !== 0)
+      if (this.random(factor) !== 0)
         return false;
     } else {
       if (!src)
@@ -563,7 +603,7 @@ class HostList {
       this.totalFresh += 1;
     }
 
-    const bucket = this.freshBucket(entry);
+    const bucket = this.freshBucket(entry, src);
 
     if (bucket.has(entry.key()))
       return false;
@@ -828,7 +868,7 @@ class HostList {
       items.push(entry);
 
     for (let i = 0; i < items.length && out.length < 2500; i++) {
-      const j = random(items.length - i);
+      const j = this.random(items.length - i);
 
       [items[i], items[i + j]] = [items[i + j], items[i]];
 
@@ -1409,6 +1449,14 @@ HostList.scores = {
 /**
  * Host Entry
  * @alias module:net.HostEntry
+ * @property {NetAddress} addr - host address.
+ * @property {NetAddress} src - the first address we discovered this entry
+ *                              from.
+ * @property {Boolean} used - is it in the used set.
+ * @property {Number} refCount - Reference count in new buckets.
+ * @property {Number} attempts - connection attempts since last successful one.
+ * @property {Number} lastSuccess - last success timestamp.
+ * @property {Number} lastAttempt - last attempt timestamp.
  */
 
 class HostEntry {
@@ -1630,6 +1678,7 @@ class HostListOptions {
     this.onion = false;
     this.brontideOnly = false;
     this.banTime = common.BAN_TIME;
+    this.random = random;
 
     this.address = new NetAddress();
     this.address.services = this.services;
@@ -1762,6 +1811,11 @@ class HostListOptions {
       this.flushInterval = options.flushInterval;
     }
 
+    if (options.random != null) {
+      assert(typeof options.random === 'function');
+      this.random = options.random;;
+    }
+
     this.address.time = this.network.now();
     this.address.services = this.services;
 
@@ -1781,65 +1835,11 @@ function random(max) {
   return Math.floor(Math.random() * max);
 }
 
-function groupKey(raw) {
-  // See: https://github.com/bitcoin/bitcoin/blob/e258ce7/src/netaddress.cpp#L413
-  // Todo: Use IP->ASN mapping, see:
-  // https://github.com/bitcoin/bitcoin/blob/adea5e1/src/addrman.h#L274
-  let type = 6; // NET_IPV6
-  let start = 0;
-  let bits = 16;
-  let i = 0;
-
-  if (IP.isLocal(raw)) {
-    type = 255; // NET_LOCAL
-    bits = 0;
-  } else if (!IP.isRoutable(raw)) {
-    type = 0; // NET_UNROUTABLE
-    bits = 0;
-  } else if (IP.isIPv4(raw) || IP.isRFC6145(raw) || IP.isRFC6052(raw)) {
-    type = 4; // NET_IPV4
-    start = 12;
-  } else if (IP.isRFC3964(raw)) {
-    type = 4; // NET_IPV4
-    start = 2;
-  } else if (IP.isRFC4380(raw)) {
-    const buf = Buffer.alloc(3);
-    buf[0] = 4; // NET_IPV4
-    buf[1] = raw[12] ^ 0xff;
-    buf[2] = raw[13] ^ 0xff;
-    return buf;
-  } else if (IP.isOnion(raw)) {
-    type = 8; // NET_ONION
-    start = 6;
-    bits = 4;
-  } else if (raw[0] === 0x20
-          && raw[1] === 0x01
-          && raw[2] === 0x04
-          && raw[3] === 0x70) {
-    bits = 36;
-  } else {
-    bits = 32;
-  }
-
-  const out = Buffer.alloc(1 + ((bits + 7) >>> 3));
-
-  out[i++] = type;
-
-  while (bits >= 8) {
-    out[i++] = raw[start++];
-    bits -= 8;
-  }
-
-  if (bits > 0)
-    out[i++] = raw[start] | ((1 << (8 - bits)) - 1);
-
-  assert(i === out.length);
-
-  return out;
-}
-
 /*
  * Expose
  */
 
-module.exports = HostList;
+exports = HostList;
+exports.HostEntry = HostEntry;
+
+module.exports = exports;