feat: Initial code

.gitignore

@@ -0,0 +1,4 @@
+__pycache__/
+.venv/
+.env
+.env.*

README.md

@@ -0,0 +1 @@
+# Explorer Database populater

indexerClasses.py

@@ -0,0 +1,137 @@
+import json
+import asyncio
+import requests
+
+class Block:
+    def __init__(self, data):
+        if isinstance(data, dict):
+            self.hash = data["hash"]
+            self.height = data["height"]
+            self.depth = data["depth"]
+            self.version = data["version"]
+            self.prevBlock = data["prevBlock"]
+            self.merkleRoot = data["merkleRoot"]
+            self.witnessRoot = data["witnessRoot"]
+            self.treeRoot = data["treeRoot"]
+            self.reservedRoot = data["reservedRoot"]
+            self.time = data["time"]
+            self.bits = data["bits"]
+            self.nonce = data["nonce"]
+            self.extraNonce = data["extraNonce"]
+            self.mask = data["mask"]
+            self.txs = data["txs"]
+        elif isinstance(data, list) or isinstance(data, tuple):
+            self.hash = data[0]
+            self.height = data[1]
+            self.depth = data[2]
+            self.version = data[3]
+            self.prevBlock = data[4]
+            self.merkleRoot = data[5]
+            self.witnessRoot = data[6]
+            self.treeRoot = data[7]
+            self.reservedRoot = data[8]
+            self.time = data[9]
+            self.bits = data[10]
+            self.nonce = data[11]
+            self.extraNonce = data[12]
+            self.mask = data[13]
+            self.txs = json.loads(data[14])
+        else:
+            raise ValueError("Invalid data type")
+
+    def __str__(self):
+        return f"Block {self.height}"
+    
+class Transaction:
+    def __init__(self, data):
+        if isinstance(data, dict):
+            self.hash = data["hash"]
+            self.witnessHash = data["witnessHash"]
+            self.fee = data["fee"]
+            self.rate = data["rate"]
+            self.mtime = data["mtime"]
+            self.block = data["block"]
+            self.index = data["index"]
+            self.version = data["version"]
+            self.inputs = data["inputs"]
+            self.outputs = data["outputs"]
+            self.locktime = data["locktime"]
+            self.hex = data["hex"]
+        elif isinstance(data, list) or isinstance(data, tuple):
+            self.hash = data[0]
+            self.witnessHash = data[1]
+            self.fee = data[2]
+            self.rate = data[3]
+            self.mtime = data[4]
+            self.block = data[5]
+            self.index = data[6]
+            self.version = data[7]
+            # Load inputs with Input class
+            self.inputs = []
+            for input in json.loads(data[8]):
+                self.inputs.append(Input(input))
+            self.outputs = []
+            for output in json.loads(data[9]):
+                self.outputs.append(Output(output))
+            self.locktime = data[10]
+            self.hex = data[11]
+        else:
+            raise ValueError("Invalid data type")
+
+    def __str__(self):
+        return f"Transaction {self.hash}"
+    
+class Input:
+    
+    def __init__(self, data):
+        if isinstance(data, dict):
+            self.prevout = data["prevout"]
+            self.witness = data["witness"]
+            self.sequence = data["sequence"]
+            self.coin = Coin(data["coin"])
+        else:
+            raise ValueError("Invalid data type")
+
+    def __str__(self):
+        return f"Input {self.prevout['hash']} {self.coin}"
+    
+class Output:
+    
+    def __init__(self, data):
+        if isinstance(data, dict):
+            self.value = data["value"]
+            self.address = data["address"]
+            self.covenant = Covenant(data["covenant"])
+        else:
+            raise ValueError("Invalid data type")
+
+    def __str__(self):
+        return f"Output {self.value} {self.address} {self.covenant}"
+    
+
+class Covenant:
+    def __init__(self, data):
+        if isinstance(data, dict):
+            self.type = data["type"]
+            self.action = data["action"]
+            self.items = data["items"]
+        else:
+            raise ValueError("Invalid data type")
+
+    def __str__(self):
+        return f"Covenant {self.type} {self.action}"
+    
+class Coin:
+    def __init__(self, data):
+        if isinstance(data, dict):
+            self.version = data["version"]
+            self.height = data["height"]
+            self.value = data["value"]
+            self.address = data["address"]
+            self.covenant = Covenant(data["covenant"])
+            self.coinbase = data["coinbase"]
+        else:
+            raise ValueError("Invalid data type")
+
+    def __str__(self):
+        return f"Coin {self.value} {self.address} {self.covenant}"

main.py

@@ -0,0 +1,318 @@
+import json
+import mysql.connector
+import requests
+from time import sleep
+import json
+import sys
+from indexerClasses import Block, Transaction
+import asyncio
+import signal
+import dotenv
+import os
+
+dotenv.load_dotenv()
+
+
+HSD_API_KEY = ""
+HSD_PORT = 12037
+HSD_IP = "127.0.0.1"
+
+if os.getenv("HSD_API_KEY"):
+    HSD_API_KEY = os.getenv("HSD_API_KEY")
+if os.getenv("HSD_PORT"):
+    HSD_PORT = os.getenv("HSD_PORT")
+if os.getenv("HSD_IP"):
+    HSD_IP = os.getenv("HSD_IP")
+
+HSD_URL = f"http://x:{HSD_API_KEY}@{HSD_IP}:{HSD_PORT}"
+if os.getenv("HSD_URL"):
+    HSD_URL = os.getenv("HSD_URL")
+
+DB_HOST = "localhost"
+DB_USER = "indexer"
+DB_PASSWORD = "supersecretpassword"
+DB_NAME = "fireindexer"
+
+if os.getenv("DB_HOST"):
+    DB_HOST = os.getenv("DB_HOST")
+if os.getenv("DB_USER"):
+    DB_USER = os.getenv("DB_USER")
+if os.getenv("DB_PASSWORD"):
+    DB_PASSWORD = os.getenv("DB_PASSWORD")
+if os.getenv("DB_NAME"):
+    DB_NAME = os.getenv("DB_NAME")
+
+
+# MySQL Database Setup
+db = mysql.connector.connect(
+    host=DB_HOST,
+    user=DB_USER,
+    password=DB_PASSWORD,
+    database=DB_NAME,
+)
+
+def indexBlock(blockHeight):
+    """Indexes a block"""
+    # Get block data
+    blockData = requests.get(f"{HSD_URL}/block/{blockHeight}")
+    if blockData.status_code != 200:
+        print(f"Error fetching block {blockHeight}: {blockData.status_code}")
+        return -1
+    print(blockHeight,end='')
+    saveBlock(blockData.json())
+    return 0
+    
+
+def saveTransaction(txData,blockHeight):
+    with db.cursor() as cursor:
+        # Check if transaction exists in database
+        cursor.execute("SELECT * FROM transactions WHERE hash = %s", (txData["hash"],))
+        txExists = cursor.fetchone()
+        if txExists:
+            print(f"\nTransaction {txData['hash']} already exists in database.")
+            return
+        
+        cursor.execute("INSERT INTO transactions (hash, witnessHash, fee, rate, mtime, block, `index`, version, inputs, outputs, locktime, hex) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", (txData["hash"], txData["witnessHash"], txData["fee"], txData["rate"], txData["mtime"], blockHeight, txData["index"], txData["version"], json.dumps(txData["inputs"]), json.dumps(txData["outputs"]), txData["locktime"], txData["hex"]))
+        db.commit()
+        print('.',end='')
+
+def saveBlock(blockData):
+    hashes = []
+    for tx in blockData["txs"]:
+        saveTransaction(tx,blockData["height"])
+        hashes.append(tx["hash"])
+
+    with db.cursor() as cursor:
+        # Check if block exists in database
+        cursor.execute("SELECT * FROM blocks WHERE height = %s", (blockData["height"],))
+        blockExists = cursor.fetchone()
+        if blockExists:
+            print(f"Block {blockData['height']} already exists in database.")
+            return
+
+        cursor.execute("INSERT INTO blocks (hash, height, depth, version, prevBlock, merkleRoot, witnessRoot, treeRoot, reservedRoot, time, bits, nonce, extraNonce, mask, txs) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", (blockData["hash"], blockData["height"], blockData["depth"], blockData["version"], blockData["prevBlock"], blockData["merkleRoot"], blockData["witnessRoot"], blockData["treeRoot"], blockData["reservedRoot"], blockData["time"], blockData["bits"], blockData["nonce"], blockData["extraNonce"], blockData["mask"], json.dumps(hashes)))
+        db.commit()
+        print('.')
+
+def setupDB():
+    """Creates the database tables"""
+    with db.cursor() as cursor:
+        cursor.execute("CREATE TABLE IF NOT EXISTS blocks (hash VARCHAR(64), height BIGINT, depth INT, version INT, prevBlock VARCHAR(64), merkleRoot VARCHAR(64), witnessRoot VARCHAR(64), treeRoot VARCHAR(64), reservedRoot VARCHAR(64), time INT, bits INT, nonce BIGINT UNSIGNED, extraNonce VARCHAR(64), mask VARCHAR(64), txs JSON)")
+        cursor.execute("CREATE TABLE IF NOT EXISTS transactions (hash VARCHAR(64), witnessHash VARCHAR(64), fee BIGINT, rate BIGINT, mtime BIGINT, block BIGINT, `index` INT, version INT, inputs JSON, outputs JSON, locktime BIGINT, hex LONGTEXT)")
+
+# Get the newest block height in the database
+def getNewestBlock() -> int:
+    """Returns the height of the newest block in the database"""
+    with db.cursor() as cursor:
+        cursor.execute("SELECT height FROM blocks ORDER BY height DESC LIMIT 1")
+        newestBlock = cursor.fetchone()
+        if newestBlock:
+            return int(newestBlock[0])
+    return -1
+        
+
+def indexChain():
+    startBlock = getNewestBlock()
+    if startBlock == -1:
+        print("ERROR GETTING NEWEST BLOCK")
+        sys.exit(1)
+
+    count = 0
+    while True:
+        if count >= 1000:
+            break
+        if indexBlock(startBlock + count) != 0:
+            break
+        count += 1
+    print(f"Indexing from block {startBlock} to {startBlock + count} complete.")
+
+def dbCheck():
+    # For the first 100 blocks, check for transactions
+    for i in range(100):
+        with db.cursor() as cursor:
+            cursor.execute("SELECT * FROM blocks WHERE height = %s", (i,))
+            block = cursor.fetchone()
+            if not block:
+                return
+            block = Block(block)
+            print(block)
+
+
+def getBlock(height):
+    with db.cursor() as cursor:
+        cursor.execute("SELECT * FROM blocks WHERE height = %s", (height,))
+        block = cursor.fetchone()
+        if not block:
+            return None
+        return Block(block)
+
+def getTransaction(hash):
+    with db.cursor() as cursor:
+        cursor.execute("SELECT * FROM transactions WHERE hash = %s", (hash,))
+        tx = cursor.fetchone()
+        if not tx:
+            return None
+        
+        return Transaction(tx)
+
+
+def getNodeHeight():
+    response = requests.get(HSD_URL)
+    if response.status_code != 200:
+        print(f"Error fetching Node Block: {response.text}")
+        return -1
+    info = response.json()
+    return info["chain"]["height"]
+
+
+def getFirstMissingBlock():
+    """Finds missing block heights in the database."""
+    with db.cursor() as cursor:
+        cursor.execute("SELECT height FROM blocks ORDER BY height ASC")
+        heights = [row[0] for row in cursor.fetchall()]
+
+    if not heights:
+        return 0
+
+    block = 0
+    for i in heights:
+        if i == block:
+            block += 1
+        else:
+            return block
+
+    return block
+
+
+
+
+
+async def main():
+    blockWatcher = BlockWatcher(HSD_URL)
+    blockWatcher.start()
+
+    # Handle Ctrl+C interrupt
+    def handle_interrupt(sig, frame):
+        print("\nReceived interrupt, stopping...")
+        blockWatcher.stop()
+
+    signal.signal(signal.SIGINT, handle_interrupt)  # Handle Ctrl+C
+
+    # Keep running until interrupted
+    while blockWatcher.running:
+        await asyncio.sleep(1)
+    print("Closing mempool watcher.", end="")
+
+    signal.signal(signal.SIGINT, signal.SIG_DFL)    
+    while blockWatcher.closing:        
+        print(".", end="")
+        await asyncio.sleep(1)        
+
+
+
+
+
+# region Classes
+class BlockWatcher:
+    def __init__(self, url ,check_interval=1):
+        self.url = url
+        self.checkInterval = check_interval
+        self.running = True
+        self.closing = False
+        self.block = 0
+
+    def start(self):
+        asyncio.create_task(self.loop())
+
+
+
+    async def loop(self):        
+        while self.running:
+            response = requests.get(self.url)
+            if response.status_code != 200:
+                print(f"Error fetching info: {response.status_code}")
+                await asyncio.sleep(self.checkInterval)
+                continue
+            info = response.json()
+            height = info["chain"]["height"]
+            if height > self.block:
+                self.block = height
+                print(f"New block: {height}")
+                if indexBlock(height) != 0:
+                    print("Error indexing block")
+
+            await asyncio.sleep(self.checkInterval)
+
+        self.closing = False
+
+    def stop(self):
+        self.closing = True
+        self.running = False        
+
+class CatchUp:
+    def __init__(self, currentHeight, targetHeight):
+        self.currentHeight = currentHeight
+        self.targetHeight = targetHeight
+        self.running = True
+        self.closing = False
+
+    async def catchUp(self):
+        print(f"Catching up from {self.currentHeight} to {self.targetHeight}")
+        def signal_handler(sig, frame):
+            self.stop()
+            print("Caught Ctrl+C")
+            
+        signal.signal(signal.SIGINT, signal_handler)
+        asyncio.create_task(self.loop())
+        while self.running:
+            await asyncio.sleep(1)
+        print("Stopping catch up")
+        while self.closing:
+            await asyncio.sleep(1)
+        print("Stopped catch up")
+
+
+
+    async def loop(self):
+        while self.running:
+            if self.currentHeight >= self.targetHeight:
+                self.running = False
+                print(f"Caught up to {self.targetHeight}")
+                return
+            
+            if indexBlock(self.currentHeight + 1) != 0:
+                print(f"Error indexing block {self.currentHeight + 1}")
+                self.running = False
+                return
+            self.currentHeight += 1
+        self.closing = False
+
+    def stop(self):
+        self.closing = True
+        self.running = False
+# endregion
+
+
+
+if __name__ == "__main__":
+    setupDB()
+    # Check if DB needs to catch up
+    newestBlock = getFirstMissingBlock()
+    NodeHeight = getNodeHeight()
+    if newestBlock == -1:
+        print("ERROR GETTING NEWEST BLOCK")
+        sys.exit(1)
+    
+    if NodeHeight == -1:
+        print("ERROR GETTING NODE HEIGHT")
+        sys.exit(1)
+
+    if newestBlock < NodeHeight:
+        print(f"Database is out of sync. Catching up from {newestBlock} to {NodeHeight}")
+        catchUpper = CatchUp(newestBlock, NodeHeight)
+        asyncio.run(catchUpper.catchUp())
+        
+
+
+    print("Starting mempool watcher.")
+    asyncio.run(main())

requirements.txt

@@ -0,0 +1,3 @@
+mysql-connector-python
+requests
+python-dotenv