Compare commits

..

9 Commits

3 changed files with 511 additions and 107 deletions

View File

@@ -42,6 +42,25 @@ class Block:
def __str__(self):
return f"Block {self.height}"
def toJSON(self) -> dict:
return {
"hash": self.hash,
"height": self.height,
"depth": self.depth,
"version": self.version,
"prevBlock": self.prevBlock,
"merkleRoot": self.merkleRoot,
"witnessRoot": self.witnessRoot,
"treeRoot": self.treeRoot,
"reservedRoot": self.reservedRoot,
"time": self.time,
"bits": self.bits,
"nonce": self.nonce,
"extraNonce": self.extraNonce,
"mask": self.mask,
"txs": self.txs
}
class Transaction:
def __init__(self, data):
if isinstance(data, dict):
@@ -67,10 +86,10 @@ class Transaction:
self.index = data[6]
self.version = data[7]
# Load inputs with Input class
self.inputs = []
self.inputs: list[Input] = []
for input in json.loads(data[8]):
self.inputs.append(Input(input))
self.outputs = []
self.outputs: list[Output] = []
for output in json.loads(data[9]):
self.outputs.append(Output(output))
self.locktime = data[10]
@@ -81,6 +100,22 @@ class Transaction:
def __str__(self):
return f"Transaction {self.hash}"
def toJSON(self) -> dict:
return {
"hash": self.hash,
"witnessHash": self.witnessHash,
"fee": self.fee,
"rate": self.rate,
"mtime": self.mtime,
"block": self.block,
"index": self.index,
"version": self.version,
"inputs": [input.toJSON() for input in self.inputs],
"outputs": [output.toJSON() for output in self.outputs],
"locktime": self.locktime,
"hex": self.hex
}
class Input:
def __init__(self, data):
@@ -88,13 +123,27 @@ class Input:
self.prevout = data["prevout"]
self.witness = data["witness"]
self.sequence = data["sequence"]
self.coin = Coin(data["coin"])
self.address = None
self.coin = None
if "address" in data:
self.address = data["address"]
if "coin" in data:
self.coin = Coin(data["coin"])
else:
raise ValueError("Invalid data type")
def __str__(self):
return f"Input {self.prevout['hash']} {self.coin}"
def toJSON(self) -> dict:
return {
"prevout": self.prevout,
"witness": self.witness,
"sequence": self.sequence,
"address": self.address,
"coin": self.coin.toJSON() if self.coin else None
}
class Output:
def __init__(self, data):
@@ -108,6 +157,20 @@ class Output:
def __str__(self):
return f"Output {self.value} {self.address} {self.covenant}"
def toJSON(self) -> dict:
return {
"value": self.value,
"address": self.address,
"covenant": self.covenant.toJSON()
}
def hex_to_ascii(hex_string):
# Convert the hex string to bytes
bytes_obj = bytes.fromhex(hex_string)
# Decode the bytes object to an ASCII string
ascii_string = bytes_obj.decode('ascii')
return ascii_string
class Covenant:
def __init__(self, data):
@@ -115,11 +178,75 @@ class Covenant:
self.type = data["type"]
self.action = data["action"]
self.items = data["items"]
self.nameHash = None
self.height = None
self.name = None
self.flags = None
self.hash = None
self.nonce = None
self.recordData = None
self.blockHash = None
self.version = None
self.Address = None
self.claimHeight = None
self.renewalCount = None
if self.type > 0: # All but NONE
self.nameHash = self.items[0]
self.height = self.items[1]
if self.type == 1: # CLAIM
self.flags = self.items[3]
if self.type in [1,2,3]: # CLAIM, OPEN, BID
self.name = hex_to_ascii(self.items[2])
if self.type == 3: # BID
self.hash = self.items[3]
if self.type == 4: # REVEAL
self.nonce = self.items[2]
if self.type in [6,7]: # REGISTER, UPDATE
self.recordData = self.items[2]
if self.type == 6: # REGISTER
self.blockHash = self.items[3]
if self.type == 8: # RENEW
self.blockHash = self.items[2]
if self.type == 9: # TRANSFER
self.version = self.items[2]
self.Address = self.items[3]
if self.type == 10: # FINALIZE
self.name = hex_to_ascii(self.items[2])
self.flags = self.items[3]
self.claimHeight= self.items[4]
self.renewalCount = self.items[5]
self.blockHash = self.items[6]
# TYPE 11 - REVOKE (Only has namehash and height)
else:
raise ValueError("Invalid data type")
def __str__(self):
return f"Covenant {self.type} {self.action}"
return self.toString()
def toString(self):
return self.action
def toJSON(self) -> dict:
return {
"type": self.type,
"action": self.action,
"items": self.items
}
class Coin:
def __init__(self, data):
@@ -135,3 +262,159 @@ class Coin:
def __str__(self):
return f"Coin {self.value} {self.address} {self.covenant}"
def toJSON(self) -> dict:
return {
"version": self.version,
"height": self.height,
"value": self.value,
"address": self.address,
"covenant": self.covenant.toJSON(),
"coinbase": self.coinbase
}
class Bid:
def __init__(self, covenant: Covenant, tx: Transaction):
self.name = covenant.name
self.nameHash = covenant.nameHash
self.height = tx.block
self.tx: Transaction = tx
self.bidHash = covenant.hash
self.bid = covenant
self.reveal = None
self.redeem = None
self.value = 0
self.blind = 0
self.txs = [tx.hash]
# TODO add blind calculation
def update(self, covenant: Covenant, tx: Transaction):
if covenant.type == 4: # REVEAL
self.reveal = covenant
self.txs.append(tx.hash)
# TODO add true bid calculation
# TODO add redeem/register covenants
class Name:
def __init__(self, data):
self.name = None
self.nameHash = None
self.state = "CLOSED"
self.height = 0
self.lastRenewal = 0
self.owner = None
self.value = 0
self.highest = 0
self.data = None
self.transfer = 0
self.revoked = 0
self.claimed = 0
self.renewals = 0
self.registered = False
self.expired = False
self.weak = False
self.stats = None
self.start = None
self.txs = []
self.bids = []
if isinstance(data, Covenant):
if not data.type in [1,2]:
print(data.type)
raise ValueError("Invalid covenant type")
self.name = data.name
self.nameHash = data.nameHash
self.height = data.height
if data.type == 2: # OPEN
self.state = "OPEN"
elif isinstance(data, dict):
for key, value in data.items():
setattr(self, key, value)
elif isinstance(data, list) or isinstance(data, tuple):
for key, value in zip(self.__dict__.keys(), data):
setattr(self, key, value)
else:
raise ValueError("Invalid data type")
def __str__(self):
return self.name
def update(self, covenant: Covenant, tx: Transaction):
self.txs.append(tx.hash)
if covenant.type == 0: # NONE
return
if covenant.type == 1: # CLAIM
self.state = "CLOSED"
self.claimed += 1
if covenant.type == 2: # OPEN
self.state = "OPEN"
self.height = covenant.height
if covenant.type == 3: # BID
bid: Bid = Bid(covenant, tx)
self.bids.append(bid)
if covenant.type == 4: # REVEAL
# Get the index of the REVEAL in the outputs
index = 0
for output in tx.outputs:
if output.covenant.hash == covenant.hash:
break
index += 1
# Get input from index
tx_input = tx.inputs[index]
# TODO get matching bid
print(tx_input)
print(covenant)
print(tx)
print(self.bids)
raise NotImplementedError
if covenant.type == 7: # UPDATE
# TODO
raise NotImplementedError
if covenant.type in [6,8]: # REGISTER, RENEW
self.lastRenewal = covenant.height
self.registered = True
if covenant.type == 6: # REGISTER
# TODO
raise NotImplementedError
if covenant.type == 9: # TRANSFER
# TODO
raise NotImplementedError
if covenant.type == 10: # FINALIZE
# TODO
raise NotImplementedError
def toJSON(self) -> dict:
return {
"name": self.name,
"nameHash": self.nameHash,
"state": self.state,
"height": self.height,
"lastRenewal": self.lastRenewal,
"owner": self.owner,
"value": self.value,
"highest": self.highest,
"data": self.data,
"transfer": self.transfer,
"revoked": self.revoked,
"claimed": self.claimed,
"renewals": self.renewals,
"registered": self.registered,
"expired": self.expired,
"weak": self.weak,
"stats": self.stats,
"start": self.start,
"txs": self.txs,
"bids": self.bids
}

316
main.py
View File

@@ -1,10 +1,11 @@
import json
import mysql.connector
from clickhouse_driver import Client
import clickhouse_connect
import requests
from time import sleep
import json
import sys
from indexerClasses import Block, Transaction
from indexerClasses import Block, Transaction, Input, Output, Covenant,Name
import asyncio
import signal
import dotenv
@@ -45,23 +46,19 @@ if os.getenv("DB_NAME"):
DB_NAME = os.getenv("DB_NAME")
# MySQL Database Setup
dbSave = mysql.connector.connect(
# Clickhouse Database Setup
dbSave = clickhouse_connect.create_client(
host=DB_HOST,
user=DB_USER,
password=DB_PASSWORD,
database=DB_NAME,
charset='utf8mb4',
collation='utf8mb4_unicode_ci',
database=DB_NAME
)
dbGet = mysql.connector.connect(
dbGet = Client(
host=DB_HOST,
user=DB_USER,
password=DB_PASSWORD,
database=DB_NAME,
charset='utf8mb4',
collation='utf8mb4_unicode_ci',
database=DB_NAME
)
def indexBlock(blockHeight):
@@ -76,32 +73,46 @@ def indexBlock(blockHeight):
return 0
def getNameFromHash(nameHash):
name = requests.post(HSD_URL, json={"method": "getnamebyhash", "params": [nameHash]})
if name.status_code != 200:
print(f"Error fetching name {nameHash}: {name.status_code}")
return -1
name = name.json()
if not name["result"]:
return -1
name = name["result"]
nameInfo = requests.post(HSD_URL, json={"method": "getnameinfo", "params": [name]})
if nameInfo.status_code != 200:
print(f"Error fetching name info {name}: {nameInfo.status_code}")
return -1
nameInfo = nameInfo.json()
if not nameInfo["result"]:
print(f"Error fetching name info {name}: {nameInfo['error']}")
return -1
return nameInfo["result"]
def saveTransactions(txList, blockHeight):
if not txList:
return
# Prepare data for batch insert
txValues = []
for txData in txList:
print('.', end='', flush=True)
txValues.append((
txValues = [
(
txData["hash"], txData["witnessHash"], txData["fee"], txData["rate"],
txData["mtime"], blockHeight, txData["index"], txData["version"],
json.dumps(txData["inputs"]), json.dumps(txData["outputs"]),
txData["locktime"], txData["hex"]
))
)
for txData in txList
]
print(f"Inserting {len(txValues)} transactions...")
return dbSave.insert("transactions", txValues, column_names=[
"hash", "witnessHash", "fee", "rate", "mtime", "block", "tx_index", "version",
"inputs", "outputs", "locktime", "hex"
])
# Bulk insert transactions
query = """
INSERT INTO transactions (hash, witnessHash, fee, rate, mtime, block, `index`, version,
inputs, outputs, locktime, hex)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON DUPLICATE KEY UPDATE hash=hash
"""
with dbSave.cursor() as cursor:
cursor.executemany(query, txValues)
dbSave.commit()
def saveBlock(blockData):
hashes = [tx["hash"] for tx in blockData["txs"]]
@@ -110,82 +121,134 @@ def saveBlock(blockData):
saveTransactions(blockData["txs"], blockData["height"])
# Insert block if it doesn't exist
query = """
INSERT INTO blocks (hash, height, depth, version, prevBlock, merkleRoot, witnessRoot,
treeRoot, reservedRoot, time, bits, nonce, extraNonce, mask, txs)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON DUPLICATE KEY UPDATE hash=hash
"""
blockValues = (
blockValues = [(
blockData["hash"], blockData["height"], blockData["depth"], blockData["version"],
blockData["prevBlock"], blockData["merkleRoot"], blockData["witnessRoot"],
blockData["treeRoot"], blockData["reservedRoot"], blockData["time"],
blockData["bits"], blockData["nonce"], blockData["extraNonce"],
blockData["mask"], json.dumps(hashes)
)
blockData["mask"], json.dumps(hashes) # Convert tx hashes to JSON string
)]
with dbSave.cursor() as cursor:
cursor.execute(query, blockValues)
dbSave.commit()
print('')
dbSave.insert("blocks", blockValues, column_names=[
"hash", "height", "depth", "version", "prevBlock", "merkleRoot", "witnessRoot",
"treeRoot", "reservedRoot", "time", "bits", "nonce", "extraNonce",
"mask", "txs"
])
def setupDB():
"""Creates the database tables"""
with dbSave.cursor() as cursor:
cursor.execute("CREATE TABLE IF NOT EXISTS blocks (hash VARCHAR(64), height BIGINT, depth INT, version INT, prevBlock VARCHAR(64), merkleRoot VARCHAR(64), witnessRoot VARCHAR(64), treeRoot VARCHAR(64), reservedRoot VARCHAR(64), time INT, bits INT, nonce BIGINT UNSIGNED, extraNonce VARCHAR(64), mask VARCHAR(64), txs JSON)")
cursor.execute("CREATE TABLE IF NOT EXISTS transactions (hash VARCHAR(64), witnessHash VARCHAR(64), fee BIGINT, rate BIGINT, mtime BIGINT, block BIGINT, `index` INT, version INT, inputs JSON, outputs JSON, locktime BIGINT, hex LONGTEXT)")
print('block saved')
# def setupDB():
# """Creates the database tables"""
# dbSave.execute("CREATE TABLE IF NOT EXISTS blocks ( hash String, height UInt64, depth Int32, version Int32, prevBlock String, merkleRoot String, witnessRoot String, treeRoot String, reservedRoot String, time UInt32, bits Int32, nonce UInt64, extraNonce String, mask String, txs String ) ENGINE = MergeTree() ORDER BY (hash, height)")
# dbSave.execute("CREATE TABLE IF NOT EXISTS transactions ( hash String, witnessHash String, fee Int64, rate Int64, mtime Int64, block UInt64, tx_index Int32, version Int32, inputs String, outputs String, locktime Int64, hex String ) ENGINE = MergeTree() ORDER BY (hash, block)")
# dbSave.execute("CREATE TABLE IF NOT EXISTS names ( name String, nameHash String, state String, height UInt64, lastRenewal Int64, owner String, value Int64, highest Int64, data String, transfer Int64, revoked Int64, claimed Int64, renewals Int64, registered UInt8, expired UInt8, weak UInt8, stats String, start String, txs String, bids String ) ENGINE = MergeTree() ORDER BY (name, height)")
# Get the newest block height in the database
def getNewestBlock() -> int:
"""Returns the height of the newest block in the database"""
dbNB = mysql.connector.connect(
host=DB_HOST,
user=DB_USER,
password=DB_PASSWORD,
database=DB_NAME,
charset='utf8mb4',
collation='utf8mb4_unicode_ci',
)
with dbNB.cursor() as cursor:
cursor.execute("SELECT height FROM blocks ORDER BY height DESC LIMIT 1")
newestBlock = cursor.fetchone()
if newestBlock:
return int(newestBlock[0])
dbNB.close()
return -1
newestBlock = dbGet.query("SELECT height FROM blocks ORDER BY height DESC LIMIT 1").result
return int(newestBlock[0][0]) if newestBlock else -1
def dbCheck():
# For the first 100 blocks, check for transactions
for i in range(100):
with dbGet.cursor() as cursor:
cursor.execute("SELECT * FROM blocks WHERE height = %s", (i,))
block = cursor.fetchone()
if not block:
return
block = Block(block)
print(block)
def getBlock(height):
with dbGet.cursor() as cursor:
cursor.execute("SELECT * FROM blocks WHERE height = %s", (height,))
block = cursor.fetchone()
block = dbGet.query(f"SELECT * FROM blocks WHERE height = {i}").result
if not block:
return None
return Block(block)
return
print(Block(block[0]))
def getTransaction(hash):
with dbGet.cursor() as cursor:
cursor.execute("SELECT * FROM transactions WHERE hash = %s", (hash,))
tx = cursor.fetchone()
if not tx:
return None
return Transaction(tx)
def getBlock(height) -> Block | None:
"""Fetch a block by height"""
block = dbGet.query(f"SELECT * FROM blocks WHERE height = {height}").result
return Block(block[0]) if block else None
def getTransaction(tx_hash) -> Transaction | None:
"""Fetch a transaction by hash"""
tx = dbGet.query(f"SELECT * FROM transactions WHERE hash = '{tx_hash}'").result
return Transaction(tx[0]) if tx else None
def getTransactions(height) -> list[Transaction] | None:
"""Fetch all transactions for a given block height"""
txs = dbGet.query(f"SELECT * FROM transactions WHERE block = {height}").result
return [Transaction(tx) for tx in txs] if txs else None
def getNameFromHash(nameHash):
"""Fetch a name record by nameHash"""
name = dbGet.query(f"SELECT * FROM names WHERE nameHash = '{nameHash}'").result
return Name(name[0]) if name else -1
def getNamesFromBlock(height):
transactions = getTransactions(height)
if not transactions:
return -1
namesToSave: list[Name] = []
names = []
for tx in transactions:
for output in tx.outputs:
cov = output.covenant
if cov.type == 0: # NONE
continue
# Check if name exists in block
if cov.nameHash in names:
for name in namesToSave:
if name.nameHash == cov.nameHash:
# Remove name from list
namesToSave.remove(name)
# Update name
name.update(cov,tx)
else:
name = getNameFromHash(cov.nameHash)
if name == -1:
# Create new name
name = Name(cov)
name.txs.append(tx.hash)
name.height = height
else:
name.update(cov,tx)
namesToSave.append(name)
queryData = []
for name in namesToSave:
nameInfo = name.toJSON()
queryData.append((
nameInfo["name"],
nameInfo["nameHash"],
nameInfo["state"],
nameInfo["height"],
nameInfo["lastRenewal"],
json.dumps(nameInfo["owner"]),
nameInfo["value"],
nameInfo["highest"],
json.dumps(nameInfo["data"]),
json.dumps(nameInfo["transfer"]),
nameInfo["revoked"],
nameInfo["claimed"],
nameInfo["renewals"],
nameInfo["registered"],
nameInfo["expired"],
nameInfo["weak"],
json.dumps(nameInfo["stats"]),
json.dumps(nameInfo["start"]),
json.dumps(nameInfo["txs"]),
json.dumps(nameInfo["bids"])
))
dbSave.insert("names", queryData, column_names=[
"name", "nameHash", "state", "height", "lastRenewal", "owner", "value", "highest",
"data", "transfer", "revoked", "claimed", "renewals", "registered", "expired",
"weak", "stats", "start", "txs", "bids"
])
return 0
def getNodeHeight():
@@ -196,24 +259,22 @@ def getNodeHeight():
info = response.json()
return info["chain"]["height"]
def getFirstMissingBlock() -> int:
"""Finds the first missing block height in the database."""
def getFirstMissingBlock():
"""Finds missing block heights in the database."""
with dbGet.cursor() as cursor:
cursor.execute("SELECT height FROM blocks ORDER BY height ASC")
heights = [row[0] for row in cursor.fetchall()]
# Fetch all existing block heights in ascending order
result = dbGet.execute("SELECT height FROM blocks ORDER BY height ASC").result
heights = [row[0] for row in result]
if not heights:
return 0
return 0 # No blocks found, start from 0
block = 0
for i in heights:
if i == block:
block += 1
else:
return block
# Find the first missing block height
for expected, actual in enumerate(heights):
if expected != actual:
return expected # First missing height found
return block
return len(heights) # No missing block, return next expected height
async def main():
@@ -270,6 +331,10 @@ class BlockWatcher:
if indexBlock(height) != 0:
print("Error indexing block")
self.block = self.block - 1
else:
# Check if there are any new names
if getNamesFromBlock(height) < 0:
print("Error indexing names")
await asyncio.sleep(self.checkInterval)
@@ -322,6 +387,50 @@ class CatchUp:
def stop(self):
self.closing = True
self.running = False
class NameSyncer:
def __init__(self, currentHeight, targetHeight):
self.currentHeight = currentHeight - 1
self.targetHeight = targetHeight
self.running = True
self.closing = False
self.interupted = False
async def sync(self):
print(f"Syncing names from {self.currentHeight} to {self.targetHeight}")
def signal_handler(sig, frame):
self.interupted = True
self.stop()
print("\n\nCaught Ctrl+C\n")
signal.signal(signal.SIGINT, signal_handler)
asyncio.create_task(self.loop())
while self.running:
await asyncio.sleep(1)
print("Stopping catch up")
while self.closing:
await asyncio.sleep(1)
print("Stopped catch up")
async def loop(self):
while self.running:
if self.currentHeight >= self.targetHeight:
self.running = False
print(f"Caught up to {self.targetHeight}")
return
if getNamesFromBlock(self.currentHeight + 1) != 0:
print(f"Error indexing names {self.currentHeight + 1}")
self.running = False
return
self.currentHeight += 1
self.closing = False
def stop(self):
self.closing = True
self.running = False
# endregion
@@ -349,7 +458,7 @@ def start_flask_in_thread():
if __name__ == "__main__":
# Webserver in background
start_flask_in_thread()
setupDB()
# setupDB()
# Check if DB needs to catch up
newestBlock = getFirstMissingBlock()
NodeHeight = getNodeHeight()
@@ -370,5 +479,16 @@ if __name__ == "__main__":
print("Starting mempool watcher.")
asyncio.run(main())
# Get names
namesyncer = NameSyncer(2000, 2025)
asyncio.run(namesyncer.sync())
if namesyncer.interupted:
sys.exit(1)
# print("Starting mempool watcher.")
# asyncio.run(main())
print("Finished")

View File

@@ -1,4 +1,5 @@
mysql-connector-python
clickhouse-driver
clickhouse-connect
requests
python-dotenv
flask