Debug stuc

experiments/honest_party_test_EC2.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python
 __author__ = 'aluex'
-from gevent import monkey
+from gevent import monkey, sleep
 monkey.patch_all()
 
 from gevent.queue import *
@@ -267,7 +267,7 @@ def toBeScheduled():
             finally:
                 print "Consensus Finished"
 
-        s = sched.scheduler(time.time, time.sleep)
+        s = sched.scheduler(time.time, sleep)
 
         time_now = time.time()
         delay = options.delaytime - time_now

experiments/run_local_tor.py

@@ -1,6 +1,7 @@
 __author__ = 'aluex'
 
 import subprocess32 as subprocess
+from gevent import sleep
 import time
 def runOnTransaction(N, t, Tx):
     retry = True
@@ -15,7 +16,7 @@ def runOnTransaction(N, t, Tx):
             retry = False
         except subprocess.TimeoutExpired:
             retry = True
-            time.sleep(2)
+            sleep(2)
     q = subprocess.check_output(['python', 'process.py', 'msglog.TorMultiple'])
     print N, t, Tx, q.replace('\n', ' ')
 

honeybadgerbft/core/binaryagreement.py

@@ -1,3 +1,5 @@
+from gevent import monkey
+monkey.patch_all()
 import gevent
 from gevent.event import Event
 

honeybadgerbft/core/commonsubset.py

@@ -1,6 +1,11 @@
+from gevent import monkey
+monkey.patch_all()
 import gevent
 
 
+from our_srcs.consts import *
+from logging import getLogger; logger=getLogger(LOGGER_NAME)
+
 def commonsubset(pid, N, f, rbc_out, aba_in, aba_out):
     """The BKR93 algorithm for asynchronous common subset.
 

honeybadgerbft/core/honeybadger.py

@@ -1,8 +1,11 @@
 from collections import namedtuple
 from enum import Enum
-
+from gevent import monkey
+monkey.patch_all()
 import gevent
 from gevent.queue import Queue
+import hashlib
+import random
 
 from honeybadgerbft.core.commoncoin import shared_coin
 from honeybadgerbft.core.binaryagreement import binaryagreement
@@ -68,7 +71,7 @@ class HoneyBadgerBFT():
     :param recv:
     """
 
-    def __init__(self, sid, pid, B, N, f, sPK, sSK, ePK, eSK, send, recv):
+    def __init__(self, sid, pid, B, N, f, sPK, sSK, ePK, eSK, send, recv, amount=-1):
         self.sid = sid
         self.pid = pid
         self.B = B
@@ -80,10 +83,12 @@ def __init__(self, sid, pid, B, N, f, sPK, sSK, ePK, eSK, send, recv):
         self.eSK = eSK
         self._send = send
         self._recv = recv
+        self.amount = amount
 
         self.round = 0  # Current block number
         self.transaction_buffer = []
         self._per_round_recv = {}  # Buffer of incoming messages
+        self.messages_seen = set()
 
     def submit_tx(self, tx):
         """Appends the given transaction to the transaction buffer.
@@ -123,16 +128,19 @@ def _recv():
         while True:
             # For each round...
             r = self.round
-            logger.debug(f"AAAAA round number: {self.round}")
-            logger.debug(f"BBBBB transaction buffer with len: {len(self.transaction_buffer)}") 
+            logger.debug(str(self.pid) + f"AAAAA round number: {self.round}")
+            logger.debug(str(self.pid) + f"BBBBB transaction buffer with len: {len(self.transaction_buffer)}") 
             if r not in self._per_round_recv:
                 self._per_round_recv[r] = Queue()
 
             # Select all the transactions (TODO: actual random selection)
             self._prepare_transaction_buffer()
-            tx_to_send = self.transaction_buffer[:self.B]
-            logger.debug(f"EEEEE transaction_buffer for id {self.pid}: {[t[:40] for t in self.transaction_buffer]}")
-            logger.debug(f"Chosen tx_to_send for {self.pid} is {tx_to_send[0][:40]}")
+            if len(self.transaction_buffer):
+                tx_to_send = self.transaction_buffer[:self.B]
+                logger.debug(str(self.pid) + f"Chosen tx_to_send for {self.pid} is {tx_to_send[0][:40]}")
+            else:
+                tx_to_send = ['']
+            logger.debug(str(self.pid) + f"EEEEE transaction_buffer for id {self.pid}: {[t[:40] for t in self.transaction_buffer]}")
 
             # TODO: Wait a bit if transaction buffer is not full
 
@@ -145,15 +153,19 @@ def _send(j, o):
             recv_r = self._per_round_recv[r].get
             new_tx = self._run_round(r, tx_to_send[0], send_r, recv_r)
             for nino in new_tx:
-                logger.debug(f'Node id {self.pid} got from his friends: {nino[:40]}')
+                if nino != b'':
+                    logger.debug(str(self.pid) + f'Node id {self.pid} got from his friends: {nino[:40]}')
+                    self.messages_seen.add(nino)
 
             # Remove all of the new transactions from the buffer
             self.transaction_buffer = [_tx for _tx in self.transaction_buffer if _tx not in [t.decode('utf-8') for t in new_tx] and _tx not in tx_to_send]
-            logger.debug(f"New transaction buffer after getting messages for id {self.pid}: {[t[:40] for t in self.transaction_buffer]}")
+            logger.debug(str(self.pid) + f"New transaction buffer after getting messages for id {self.pid}: {[t[:40] for t in self.transaction_buffer]}")
 
             self.round += 1     # Increment the round
-            if not self.transaction_buffer:
-                logger.debug(f"CCCCC finished with rounds: {self.round}")
+            # if not self.transaction_buffer:
+            logger.debug(f"{self.pid} Up to now: {len(set(self.messages_seen))} out of {self.amount}")
+            if len(set(self.messages_seen)) == self.amount:
+                logger.debug(str(self.pid) + f"CCCCC finished with rounds: {self.round}")
                 break   # Only run one round for now
 
     def _run_round(self, r, tx_to_send, send, recv):
@@ -236,27 +248,23 @@ def rbc_send(k, o):
         # N instances of ABA, RBC
         for j in range(N):
             _setup(j)
-
         # One instance of TPKE
         def tpke_bcast(o):
             """Threshold encryption broadcast."""
             broadcast(('TPKE', 0, o))
 
         tpke_recv = Queue()
-
         # One instance of ACS
         acs = gevent.spawn(commonsubset, pid, N, f, rbc_outputs,
                            [_.put_nowait for _ in aba_inputs],
                            [_.get for _ in aba_outputs])
-
         recv_queues = BroadcastReceiverQueues(
             ACS_COIN=coin_recvs,
             ACS_ABA=aba_recvs,
             ACS_RBC=rbc_recvs,
             TPKE=tpke_recv,
         )
         gevent.spawn(broadcast_receiver_loop, recv, recv_queues)
-
         _input = Queue(1)
         _input.put(tx_to_send)
         return honeybadger_block(pid, self.N, self.f, self.ePK, self.eSK,
@@ -268,15 +276,30 @@ def tpke_bcast(o):
 def permute_list(lst, index):
     return lst[index:] + lst[:index]
 
+def sha256(s):
+    return hashlib.sha256(s.encode()).digest().hex()
+
+def distance(frac):
+    def dist(s):
+        return abs(int((2**256) * frac)  - int.from_bytes(hashlib.sha256(s.encode()).digest(), 'big'))
+    return dist
+
 class ImprovedHoneyBadgerBFT(HoneyBadgerBFT):
     def _prepare_transaction_buffer(self):
-        self.transaction_buffer = sorted(self.transaction_buffer, key=str.lower, reverse=(self.pid%2 == 1))
+        self.transaction_buffer = sorted(self.transaction_buffer, key=sha256, reverse=(self.pid%2 == 1))
 
 class PermutedHoneyBadgerBFT(HoneyBadgerBFT):
     def _prepare_transaction_buffer(self):
-        self.transaction_buffer = sorted(self.transaction_buffer, key=str.lower)
-        self.transaction_buffer = permute_list(self.transaction_buffer, self.pid)
+        self.transaction_buffer = sorted(self.transaction_buffer, key=sha256)
+        self.transaction_buffer = permute_list(self.transaction_buffer, int((self.pid / self.N) * self.N))
+        logger.debug("permutating")
 
 class RandomizedHoneyBadgerBFT(HoneyBadgerBFT):
-    def _prepare_transaction(self):
+    def _prepare_transaction_buffer(self):
         self.transaction_buffer = random.sample(self.transaction_buffer, len(self.transaction_buffer))
+
+class DistanceHoneyBadgerBFT(HoneyBadgerBFT):
+    def _prepare_transaction_buffer(self):
+        self.transaction_buffer = sorted(self.transaction_buffer, key=distance(self.pid / self.N))
+        logger.debug("permutating with distance")
+

honeybadgerbft/core/honeybadger_block.py

@@ -1,5 +1,7 @@
 from ..crypto.threshenc import tpke
 import os
+from our_srcs.consts import *
+from logging import getLogger; logger=getLogger(LOGGER_NAME)
 
 
 def serialize_UVW(U, V, W):
@@ -70,7 +72,6 @@ def honeybadger_block(pid, N, f, PK, SK, propose_in, acs_in, acs_out, tpke_bcast
         share = SK.decrypt_share(*tkey)
         # share is of the form: U_i, an element of group1
         my_shares.append(share)
-
     tpke_bcast(my_shares)
 
     # Receive everyone's shares
@@ -82,7 +83,6 @@ def honeybadger_block(pid, N, f, PK, SK, propose_in, acs_in, acs_out, tpke_bcast
             print('Received a duplicate decryption share from', j)
             continue
         shares_received[j] = shares
-
     assert len(shares_received) >= f+1
     # TODO: Accountability
     # If decryption fails at this point, we will have evidence of misbehavior,
@@ -100,5 +100,4 @@ def honeybadger_block(pid, N, f, PK, SK, propose_in, acs_in, acs_out, tpke_bcast
         plain = tpke.decrypt(key, ciph)
         decryptions.append(plain)
     # print 'Done!', decryptions
-
     return tuple(decryptions)

misc/includeTransaction.py

@@ -1,6 +1,7 @@
 __author__ = 'aluex' 
-
-from gevent import Greenlet
+from gevent import monkey
+monkey.patch_all()
+from gevent import Greenlet, sleep
 from gevent.queue import Queue, Empty
 from bkr_acs import acs
 from utils import mylog, MonitoredInt, callBackWrap, greenletFunction, \
@@ -321,7 +322,7 @@ def listener():
                 broadcast(eval(msg))  # now the msg is something we mannually send
             mylog("timestampB (%d, %lf)" % (pid, time.time()), verboseLevel=-2)
             if len(transactionCache) < B:  # Let's wait for many transactions. : )
-                time.sleep(0.5)
+                sleep(0.5)
                 print "Not enough transactions", len(transactionCache)
                 continue
 

our_srcs/analyze_results.py

@@ -3,9 +3,9 @@
 import pandas
 import re
 
-RESULT_REGEX = "\(\d\d:\d\d:\d\d\) CRITICAL: Result: (?P<result_value>\d\d.\d) \(params (?P<HB>[a-zA-Z]* [a-zA-Z]*), (?P<N>\d*), (?P<id>\d*), (?P<sz>\d*)\)"
+RESULT_REGEX = "\(\d\d:\d\d:\d\d\) CRITICAL: Result: (?P<result_value>[\d]+.?[\d]*) \(params (?P<HB>[a-zA-Z]* [a-zA-Z]*), (?P<N>\d*), (?P<id>\d*), (?P<sz>\d*)\)"
 
-SET_PARAMS = {"N": 6, "id": 2, "sz": 2}#1024*1024}
+SET_PARAMS = {"N": 10, "id": 6, "sz": 2**20}
 PARAMS = ['N', 'id', 'sz']
 
 

our_srcs/consts.py

@@ -4,11 +4,11 @@
 
 LOG_PATH = LOG_DIR + '/honeybadger_test_{}_{}_{}_{}_{}_{}.log'
 LOGGER_NAME = 'honeybadger_test_logger'
-NUM_OF_NODE_OPTIONS = [4, 6]
-NUM_OF_IDENTICAL_INPUTS_OPTIONS = [0, 2, 4, 6]
-
-NUM_OF_INPUTS_IN_ITERATION = 3
-INPUT_SIZES = [2, 1024, 1024*1024]
-SET_NUM_OF_NODES = 6
-SET_NUM_OF_IDENTICAL_INPUTS = 2
-SET_INPUT_SIZE = 2 # TODO: Change to 1024*1024
+NUM_OF_NODE_OPTIONS = [4, 6, 8, 10]
+NUM_OF_IDENTICAL_INPUTS_OPTIONS = [0, 2, 4, 6, 8, 10]
+
+NUM_OF_INPUTS_IN_ITERATION = 6
+INPUT_SIZES = [2, 2**2, 2**4, 2**8, 2**16, 2**20, 2**24]
+SET_NUM_OF_NODES = 10
+SET_NUM_OF_IDENTICAL_INPUTS = 6
+SET_INPUT_SIZE = 2**16

our_srcs/utils.py

@@ -4,13 +4,16 @@
 import datetime
 import random
 import math
-
+from gevent import monkey
+monkey.patch_all()
 from honeybadgerbft.crypto.threshsig.boldyreva import dealer
 from honeybadgerbft.crypto.threshenc import tpke
 import gevent
 from gevent.event import Event
 from gevent.queue import Queue
 
+from our_srcs.consts import *
+from logging import getLogger; logger=getLogger(LOGGER_NAME)
 
 def setup_logging():
     if not os.path.isdir(LOG_DIR):
@@ -34,7 +37,7 @@ def setup_logging():
     logger.addHandler(ch)
     logger.addHandler(fi)
 
-def setup_honeybadgers(honeybadger_class, N):
+def setup_honeybadgers(honeybadger_class, N, amount=-1):
     sid = 'sidA'
     # Generate threshold sig keys
     sPK, sSKs = dealer(N, 2, seed=None)
@@ -51,7 +54,7 @@ def setup_honeybadgers(honeybadger_class, N):
     for i in range(N):
         badgers[i] = honeybadger_class(sid, i, 1, N, 1,
                                     sPK, sSKs[i], ePK, eSKs[i],
-                                    sends[i], recvs[i])
+                                    sends[i], recvs[i], amount=amount)
         threads[i] = gevent.spawn(badgers[i].run)
     return badgers, threads
 
@@ -69,7 +72,8 @@ def simple_router(N, maxdelay=0.005, seed=None):
     def makeSend(i):
         def _send(j, o):
             delay = rnd.random() * maxdelay
-            delay *= math.log(len(o))
+            messageLen = sum([len(s) if type(s) is str else 0 for s in o[1][2]])
+            delay *= math.log(messageLen) if messageLen != 0 else 1
             gevent.spawn_later(delay, queues[j].put_nowait, (i,o))
         return _send
 

test/my_test.py

@@ -2,7 +2,8 @@
 from collections import defaultdict
 import math
 import datetime
-
+from gevent import monkey
+monkey.patch_all()
 import gevent
 from gevent.event import Event
 from gevent.queue import Queue
@@ -12,8 +13,8 @@
 from our_srcs.consts import *
 from our_srcs.utils import *
 
-from honeybadgerbft.core.honeybadger import HoneyBadgerBFT, ImprovedHoneyBadgerBFT, PermutedHoneyBadgerBFT, RandomizedHoneyBadgerBFT
-HONEYBADGERS = [("Ordered Honeybadger", HoneyBadgerBFT), ("Permuted Honeybadger", PermutedHoneyBadgerBFT),  ("Randomized Honeybadger", RandomizedHoneyBadgerBFT), ("Parity Honeybadger", ImprovedHoneyBadgerBFT)]
+from honeybadgerbft.core.honeybadger import HoneyBadgerBFT, ImprovedHoneyBadgerBFT, PermutedHoneyBadgerBFT, RandomizedHoneyBadgerBFT, DistanceHoneyBadgerBFT
+HONEYBADGERS = [("Ordered Honeybadger", HoneyBadgerBFT), ("Permuted Honeybadger", PermutedHoneyBadgerBFT),  ("Randomized Honeybadger", RandomizedHoneyBadgerBFT), ("Parity Honeybadger", ImprovedHoneyBadgerBFT), ("Distance Honeybadger", DistanceHoneyBadgerBFT)]
 setup_logging()
 logger = getLogger(LOGGER_NAME)
 
@@ -32,32 +33,37 @@ def test_honeybadger_full(HB, N, identical_inputs, input_sizes):
 
     logger.info(f"Running Honeybadger test with parameters:\n\tHoneyBadger: {HB[0]}\n\tNumber of Nodes: {N}\n\tNumber of Identical Inputs: {identical_inputs}\n\tInput Sizes: {input_sizes}")
 
-    badgers, threads = setup_honeybadgers(HB[1], N)
-
-    time_at_start = datetime.datetime.now().timestamp()
-
+    txs_to_submit = [[] for i in range(N)]
     for iter_index in range(NUM_OF_INPUTS_IN_ITERATION):
         identical_hbs = random.sample(range(N), identical_inputs)
         logger.debug(f"At epoch {iter_index} chose identical_inputs {identical_hbs}")
         for node_index in range(N):
             if node_index in identical_hbs:
-                badgers[node_index].submit_tx(f'<HBBFT Input Epoch {iter_index} Identical Input> ' + 'a'*input_sizes)
+                txs_to_submit[node_index].append(f'<HBBFT Input Epoch {iter_index} Identical Input> ' + 'a'*input_sizes)
             else:
-                badgers[node_index].submit_tx(f'<HBBFT Input Epoch {iter_index} Different Input {node_index} ' + 'a'*input_sizes)
+                txs_to_submit[node_index].append(f'<HBBFT Input Epoch {iter_index} Different Input {node_index} ' + 'a'*input_sizes)
+    amount_of_distinct_messages = len(set([msg for l in txs_to_submit for msg in l]))
+    logger.debug(f"Number of distrinct message is {amount_of_distinct_messages}")
 
+    badgers, threads = setup_honeybadgers(HB[1], N, amount_of_distinct_messages)
+
+    time_at_start = datetime.datetime.now().timestamp()
+
+    for node_index in range(N):
+        for tx in txs_to_submit[node_index]:
+            badgers[node_index].submit_tx(tx)
 
     logger.debug("Done submitting all inputs")
 
     try:
         outs = [threads[i].get() for i in range(N)]
-
         # Consistency check
         assert len(set(outs)) == 1
 
         time_at_end = datetime.datetime.now().timestamp()
         time_diff = time_at_end - time_at_start
         logger.info(f"Time passed: {time_diff}")
-        result = str(time_diff)[:4]
+        result = str(round(time_diff, 2))
         logger.critical(f"Result: {result} (params {HB[0]}, {N}, {identical_inputs}, {input_sizes})")
         return str(time_diff)[:4]