nathanwoodburn/fireexplorer-indexer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: Remove cursors

Browse Source
This commit is contained in:
Nathan Woodburn 2025-02-09 21:56:18 +11:00
parent 6d6eae297d
commit 1e783c9775
Signed by: nathanwoodburn
GPG Key ID: 203B000478AD0EF1
2 changed files with 44 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
indexerClasses.py
View File

@ -228,6 +228,9 @@ class Covenant:
self.renewalCount = self.items[5] self.renewalCount = self.items[5]
self.blockHash = self.items[6] self.blockHash = self.items[6]
# TYPE 11 - REVOKE (Only has namehash and height)
else: else:
raise ValueError("Invalid data type") raise ValueError("Invalid data type")

128
main.py
View File

@ -110,13 +110,10 @@ def saveTransactions(txList, blockHeight):
query = """ query = """
INSERT INTO transactions (hash, witnessHash, fee, rate, mtime, block, tx_index, version, INSERT INTO transactions (hash, witnessHash, fee, rate, mtime, block, tx_index, version,
inputs, outputs, locktime, hex) inputs, outputs, locktime, hex)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) VALUES
ON DUPLICATE KEY UPDATE hash=hash
""" """
return dbSave.execute(query, txValues)
with dbSave.cursor() as cursor:
cursor.executemany(query, txValues)
dbSave.commit()
def saveBlock(blockData): def saveBlock(blockData):
hashes = [tx["hash"] for tx in blockData["txs"]] hashes = [tx["hash"] for tx in blockData["txs"]]
@ -128,8 +125,7 @@ def saveBlock(blockData):
query = """ query = """
INSERT INTO blocks (hash, height, depth, version, prevBlock, merkleRoot, witnessRoot, INSERT INTO blocks (hash, height, depth, version, prevBlock, merkleRoot, witnessRoot,
treeRoot, reservedRoot, time, bits, nonce, extraNonce, mask, txs) treeRoot, reservedRoot, time, bits, nonce, extraNonce, mask, txs)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) VALUES
ON DUPLICATE KEY UPDATE hash=hash
""" """
blockValues = ( blockValues = (
@ -140,18 +136,15 @@ def saveBlock(blockData):
blockData["mask"], json.dumps(hashes) blockData["mask"], json.dumps(hashes)
) )
with dbSave.cursor() as cursor: dbSave.execute(query, blockValues)
cursor.execute(query, blockValues)
dbSave.commit()
print('') print('')
def setupDB(): # def setupDB():
"""Creates the database tables""" # """Creates the database tables"""
with dbSave.cursor() as cursor:
cursor.execute("CREATE TABLE IF NOT EXISTS blocks ( hash String, height UInt64, depth Int32, version Int32, prevBlock String, merkleRoot String, witnessRoot String, treeRoot String, reservedRoot String, time UInt32, bits Int32, nonce UInt64, extraNonce String, mask String, txs String ) ENGINE = MergeTree() ORDER BY (hash, height)") # dbSave.execute("CREATE TABLE IF NOT EXISTS blocks ( hash String, height UInt64, depth Int32, version Int32, prevBlock String, merkleRoot String, witnessRoot String, treeRoot String, reservedRoot String, time UInt32, bits Int32, nonce UInt64, extraNonce String, mask String, txs String ) ENGINE = MergeTree() ORDER BY (hash, height)")
cursor.execute("CREATE TABLE IF NOT EXISTS transactions ( hash String, witnessHash String, fee Int64, rate Int64, mtime Int64, block UInt64, tx_index Int32, version Int32, inputs String, outputs String, locktime Int64, hex String ) ENGINE = MergeTree() ORDER BY (hash, block)") # dbSave.execute("CREATE TABLE IF NOT EXISTS transactions ( hash String, witnessHash String, fee Int64, rate Int64, mtime Int64, block UInt64, tx_index Int32, version Int32, inputs String, outputs String, locktime Int64, hex String ) ENGINE = MergeTree() ORDER BY (hash, block)")
cursor.execute("CREATE TABLE IF NOT EXISTS names ( name String, nameHash String, state String, height UInt64, lastRenewal Int64, owner String, value Int64, highest Int64, data String, transfer Int64, revoked Int64, claimed Int64, renewals Int64, registered UInt8, expired UInt8, weak UInt8, stats String, start String, txs String, bids String ) ENGINE = MergeTree() ORDER BY (name, height)") # dbSave.execute("CREATE TABLE IF NOT EXISTS names ( name String, nameHash String, state String, height UInt64, lastRenewal Int64, owner String, value Int64, highest Int64, data String, transfer Int64, revoked Int64, claimed Int64, renewals Int64, registered UInt8, expired UInt8, weak UInt8, stats String, start String, txs String, bids String ) ENGINE = MergeTree() ORDER BY (name, height)")
# Get the newest block height in the database # Get the newest block height in the database
def getNewestBlock() -> int: def getNewestBlock() -> int:
@ -161,65 +154,50 @@ def getNewestBlock() -> int:
host=DB_HOST, host=DB_HOST,
user=DB_USER, user=DB_USER,
password=DB_PASSWORD, password=DB_PASSWORD,
database=DB_NAME, database=DB_NAME
charset='utf8mb4',
collation='utf8mb4_unicode_ci',
) )
with dbNB.cursor() as cursor: newestBlock = dbNB.execute("SELECT height FROM blocks ORDER BY height DESC LIMIT 1")
cursor.execute("SELECT height FROM blocks ORDER BY height DESC LIMIT 1") if newestBlock:
newestBlock = cursor.fetchone() return int(newestBlock[0])
if newestBlock:
return int(newestBlock[0])
dbNB.close()
return -1 return -1
def dbCheck(): def dbCheck():
# For the first 100 blocks, check for transactions # For the first 100 blocks, check for transactions
for i in range(100): for i in range(100):
with dbGet.cursor() as cursor: block = dbGet.execute("SELECT * FROM blocks WHERE height = %s", (i,))
cursor.execute("SELECT * FROM blocks WHERE height = %s", (i,)) if not block:
block = cursor.fetchone() return
if not block: block = Block(block)
return print(block)
block = Block(block)
print(block)
def getBlock(height) -> Block | None: def getBlock(height) -> Block | None:
with dbGet.cursor() as cursor: block = dbGet.execute("SELECT * FROM blocks WHERE height = %s", (height,))
cursor.execute("SELECT * FROM blocks WHERE height = %s", (height,))
block = cursor.fetchone() if not block:
if not block: return None
return None return Block(block)
return Block(block)
def getTransaction(hash) -> Transaction | None: def getTransaction(hash) -> Transaction | None:
with dbGet.cursor() as cursor: tx = dbGet.execute("SELECT * FROM transactions WHERE hash = %s", (hash,))
cursor.execute("SELECT * FROM transactions WHERE hash = %s", (hash,)) if not tx:
tx = cursor.fetchone() return None
if not tx: return Transaction(tx)
return None
return Transaction(tx)
def getTransactions(height) -> list[Transaction] | None: def getTransactions(height) -> list[Transaction] | None:
with dbGet.cursor() as cursor: txs = dbGet.execute("SELECT * FROM transactions WHERE block = %s", (height,))
cursor.execute("SELECT * FROM transactions WHERE block = %s", (height,)) if not txs:
txs = cursor.fetchall() return None
if not txs:
return None
# Convert to list of Transaction objects # Convert to list of Transaction objects
return [Transaction(tx) for tx in txs] return [Transaction(tx) for tx in txs]
def getNameFromHash(nameHash): def getNameFromHash(nameHash):
# Connect to db # Connect to db
with dbGet.cursor() as cursor: name = dbGet.execute("SELECT * FROM names WHERE nameHash = %s", (nameHash,))
cursor.execute("SELECT * FROM names WHERE nameHash = %s", (nameHash,)) if not name:
name = cursor.fetchone() return -1
if not name: return Name(name)
return -1
return Name(name)
def getNamesFromBlock(height): def getNamesFromBlock(height):
transactions = getTransactions(height) transactions = getTransactions(height)
@ -282,32 +260,9 @@ def getNamesFromBlock(height):
query = """ query = """
INSERT INTO names (name, nameHash, state, height, lastRenewal, owner, value, highest, data, transfer, revoked, claimed, renewals, registered, expired, weak, stats, start, txs, bids) INSERT INTO names (name, nameHash, state, height, lastRenewal, owner, value, highest, data, transfer, revoked, claimed, renewals, registered, expired, weak, stats, start, txs, bids)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) VALUES
ON DUPLICATE KEY UPDATE
state=VALUES(state),
height=VALUES(height),
lastRenewal=VALUES(lastRenewal),
owner=VALUES(owner),
value=VALUES(value),
highest=VALUES(highest),
data=VALUES(data),
transfer=VALUES(transfer),
revoked=VALUES(revoked),
claimed=VALUES(claimed),
renewals=VALUES(renewals),
registered=VALUES(registered),
expired=VALUES(expired),
weak=VALUES(weak),
stats=VALUES(stats),
start=VALUES(start),
txs=VALUES(txs),
bids=VALUES(bids)
""" """
with dbSave.cursor() as cursor: return dbSave.execute(query, queryData)
cursor.executemany(query, queryData)
dbSave.commit()
return 0
@ -322,9 +277,8 @@ def getNodeHeight():
def getFirstMissingBlock(): def getFirstMissingBlock():
"""Finds missing block heights in the database.""" """Finds missing block heights in the database."""
with dbGet.cursor() as cursor: height = dbGet.execute("SELECT height FROM blocks ORDER BY height ASC")
cursor.execute("SELECT height FROM blocks ORDER BY height ASC") heights = [row[0] for row in height]
heights = [row[0] for row in cursor.fetchall()]
if not heights: if not heights:
return 0 return 0
@ -394,7 +348,7 @@ class BlockWatcher:
self.block = self.block - 1 self.block = self.block - 1
else: else:
# Check if there are any new names # Check if there are any new names
if getNamesFromBlock(height) != 0: if getNamesFromBlock(height) < 0:
print("Error indexing names") print("Error indexing names")
await asyncio.sleep(self.checkInterval) await asyncio.sleep(self.checkInterval)
@ -519,7 +473,7 @@ def start_flask_in_thread():
if __name__ == "__main__": if __name__ == "__main__":
# Webserver in background # Webserver in background
start_flask_in_thread() start_flask_in_thread()
setupDB() # setupDB()
# Check if DB needs to catch up # Check if DB needs to catch up
newestBlock = getFirstMissingBlock() newestBlock = getFirstMissingBlock()
NodeHeight = getNodeHeight() NodeHeight = getNodeHeight()