added genesis start up and continue from last block, huey added for cronjob /scheduling WIP

Author: 2hoursleep

main.py

@@ -1,10 +1,14 @@
 from packages.block_collector import IrohaBlockAPI
 from packages.models import Block_V1
 from packages.db import session
+from packages.cronjob import huey
 import os
 import json
 import asyncio
 
+
+huey.start()
+
 private_key = os.getenv(
     "IROHA_V1_API_SECRET",
     "164879bd94411cb7c88308b6423f7dde1e6df19a98961c8aebf7d70990ade5b7",
@@ -14,9 +18,10 @@
 iroha_block_api = IrohaBlockAPI(
     creator_account=api_user, private_key=private_key, iroha_host=iroha_host,
 )
-
-def parse_iroha_blocks_to_db(height:int = 1):
+@huey.task()
+def parse_iroha_blocks_to_db(height:int = 1) -> None:
     """
+    Iroha Block Parser
     :param height: Address of the Sender,
     :param return_block: Returns block in JSON Format
     :return: None if return_block is false
@@ -47,39 +52,53 @@ def parse_iroha_blocks_to_db(height:int = 1):
         print(error)
 
 
-def async_parse_iroha_blocks_to_db(height:int = 1):
+def parse_genesis_iroha_blocks_to_db(height:int = 1):
     """
     
     """
     try:
         raw_block = iroha_block_api.get_blocks(height=height)
         block = json.loads(raw_block)
-        payload = block["blockResponse"]["block"]["blockV1"]["payload"]["transactions"][0][
-            "payload"
-        ]["reducedPayload"]["commands"]
-        block_hash = block["blockResponse"]["block"]["blockV1"]["payload"]["prevBlockHash"]
+        transactions = block["blockResponse"]["block"]["blockV1"]["payload"]["transactions"]
+        prev_block_hash = block["blockResponse"]["block"]["blockV1"]["payload"]["prevBlockHash"]
         height = block["blockResponse"]["block"]["blockV1"]["payload"]["height"]
         txNumber = block["blockResponse"]["block"]["blockV1"]["payload"]["txNumber"]
-
-        iroha_block = Block_V1.add_block(
+        Block_V1.add_block(
+            created_time='0',
             height=height,
-            block_hash=block_hash,
-            payload=payload,
-            tx_number=txNumber
-        )
-        print(iroha_block)
+            prev_block_hash=prev_block_hash,
+            transactions=transactions,
+            rejected_transactions_hashes=[],
+            signatures=[])
     except Exception as error:
         print(error)
 
-def load_blocks_to_database():
-    import time
 
-    start_time = time.time()
-    cpu_start_time = time.process_time()
+def start_up():
+    # get last block from db if none start from 1
+    try:
+        parse_genesis_iroha_blocks_to_db()
+    except:
+        print('Genesis block already exists')
+    finally:
+        last_block = Block_V1.get_last_block()
+        return last_block
+    
+
+#res = load_blocks_to_database.schedule(args=(10, 20), delay=2)
+#print(res())
+
+# Stop the scheduler. Not strictly necessary, but a good idea.
+
+def start_parser(last_height: int, scan_range: int =100):
+    curent_height = last_height
+    end_height = curent_height + scan_range
+    while curent_height < end_height:
+        parse_iroha_blocks_to_db(height=curent_height)
+        curent_height += 1
+        print(f'parsed block {curent_height}')
 
-    for i in range(1, 350):
-        parse_iroha_blocks_to_db(height=i)
-        print(f'parsed block {i}')
-    #total_real_time = time.time() - start_time
-    total_cpu_time = time.process_time() - cpu_start_time
-    print(total_cpu_time)
+last_block = start_up()
+last_height = last_block["height"]
+start_parser(last_height,100)
+huey.stop()

packages/cronjob.py

@@ -0,0 +1,14 @@
+#add huey or mini huey
+
+
+from gevent import monkey; monkey.patch_all()
+import gevent
+
+from huey.contrib.mini import MiniHuey
+
+
+huey = MiniHuey()
+
+# If we want to support scheduling tasks for execution in the future, or for
+# periodic execution (e.g. cron), then we need to call `huey.start()` which
+# starts a scheduler thread.

packages/models.py

@@ -35,25 +35,34 @@ def __repr__(self):
     def add_block(prev_block_hash, created_time ,height, transactions, rejected_transactions_hashes, signatures):
         # add Iroha block to database
         block = Block_V1(prev_block_hash, created_time ,height, transactions, rejected_transactions_hashes, signatures)
-        session.add(block)
-        session.commit()
+        
+        try:
+            session.add(block)
+            session.commit()
+        except:
+            session.rollback()
+            raise
+        finally:
+            session.close()
 
     @staticmethod
     def get_block_by_height(height: int = 1,):
         # check whether auth token has been blacklisted
         block = session.query(Block_V1).filter_by(height=height)
         return block.__dict__
 
+
     @staticmethod
     def get_block_by_hash(block_hash):
         # check whether auth token has been blacklisted
         block = session.query(Block_V1).filter_by(height=block_hash)
         return block.__dict__
 
     @staticmethod
-    def get_tx_by_command(command: str):
+    def get_last_block():
         # check whether auth token has been blacklisted
-        block = session.query(Block_V1).filter_by(height=command)
+        block = session.query(Block_V1).order_by(Block_V1.height.desc()).first()
+        print(block.height)
         return block.__dict__