fix: Update some db inserts
This commit is contained in:
parent
1e783c9775
commit
0012ecc77f
59
main.py
59
main.py
@ -1,5 +1,6 @@
|
|||||||
import json
|
import json
|
||||||
from clickhouse_driver import Client
|
from clickhouse_driver import Client
|
||||||
|
import clickhouse_connect
|
||||||
import requests
|
import requests
|
||||||
from time import sleep
|
from time import sleep
|
||||||
import json
|
import json
|
||||||
@ -46,7 +47,7 @@ if os.getenv("DB_NAME"):
|
|||||||
|
|
||||||
|
|
||||||
# Clickhouse Database Setup
|
# Clickhouse Database Setup
|
||||||
dbSave = Client(
|
dbSave = clickhouse_connect.create_client(
|
||||||
host=DB_HOST,
|
host=DB_HOST,
|
||||||
user=DB_USER,
|
user=DB_USER,
|
||||||
password=DB_PASSWORD,
|
password=DB_PASSWORD,
|
||||||
@ -96,23 +97,21 @@ def saveTransactions(txList, blockHeight):
|
|||||||
return
|
return
|
||||||
|
|
||||||
# Prepare data for batch insert
|
# Prepare data for batch insert
|
||||||
txValues = []
|
txValues = [
|
||||||
for txData in txList:
|
(
|
||||||
print('.', end='', flush=True)
|
|
||||||
txValues.append((
|
|
||||||
txData["hash"], txData["witnessHash"], txData["fee"], txData["rate"],
|
txData["hash"], txData["witnessHash"], txData["fee"], txData["rate"],
|
||||||
txData["mtime"], blockHeight, txData["index"], txData["version"],
|
txData["mtime"], blockHeight, txData["index"], txData["version"],
|
||||||
json.dumps(txData["inputs"]), json.dumps(txData["outputs"]),
|
json.dumps(txData["inputs"]), json.dumps(txData["outputs"]),
|
||||||
txData["locktime"], txData["hex"]
|
txData["locktime"], txData["hex"]
|
||||||
))
|
)
|
||||||
|
for txData in txList
|
||||||
|
]
|
||||||
|
print(f"Inserting {len(txValues)} transactions...")
|
||||||
|
return dbSave.insert("transactions", txValues, column_names=[
|
||||||
|
"hash", "witnessHash", "fee", "rate", "mtime", "block", "tx_index", "version",
|
||||||
|
"inputs", "outputs", "locktime", "hex"
|
||||||
|
])
|
||||||
|
|
||||||
# Bulk insert transactions
|
|
||||||
query = """
|
|
||||||
INSERT INTO transactions (hash, witnessHash, fee, rate, mtime, block, tx_index, version,
|
|
||||||
inputs, outputs, locktime, hex)
|
|
||||||
VALUES
|
|
||||||
"""
|
|
||||||
return dbSave.execute(query, txValues)
|
|
||||||
|
|
||||||
|
|
||||||
def saveBlock(blockData):
|
def saveBlock(blockData):
|
||||||
@ -122,22 +121,22 @@ def saveBlock(blockData):
|
|||||||
saveTransactions(blockData["txs"], blockData["height"])
|
saveTransactions(blockData["txs"], blockData["height"])
|
||||||
|
|
||||||
# Insert block if it doesn't exist
|
# Insert block if it doesn't exist
|
||||||
query = """
|
blockValues = [(
|
||||||
INSERT INTO blocks (hash, height, depth, version, prevBlock, merkleRoot, witnessRoot,
|
|
||||||
treeRoot, reservedRoot, time, bits, nonce, extraNonce, mask, txs)
|
|
||||||
VALUES
|
|
||||||
"""
|
|
||||||
|
|
||||||
blockValues = (
|
|
||||||
blockData["hash"], blockData["height"], blockData["depth"], blockData["version"],
|
blockData["hash"], blockData["height"], blockData["depth"], blockData["version"],
|
||||||
blockData["prevBlock"], blockData["merkleRoot"], blockData["witnessRoot"],
|
blockData["prevBlock"], blockData["merkleRoot"], blockData["witnessRoot"],
|
||||||
blockData["treeRoot"], blockData["reservedRoot"], blockData["time"],
|
blockData["treeRoot"], blockData["reservedRoot"], blockData["time"],
|
||||||
blockData["bits"], blockData["nonce"], blockData["extraNonce"],
|
blockData["bits"], blockData["nonce"], blockData["extraNonce"],
|
||||||
blockData["mask"], json.dumps(hashes)
|
blockData["mask"], json.dumps(hashes) # Convert tx hashes to JSON string
|
||||||
)
|
)]
|
||||||
|
|
||||||
dbSave.execute(query, blockValues)
|
|
||||||
print('')
|
dbSave.insert("blocks", blockValues, column_names=[
|
||||||
|
"hash", "height", "depth", "version", "prevBlock", "merkleRoot", "witnessRoot",
|
||||||
|
"treeRoot", "reservedRoot", "time", "bits", "nonce", "extraNonce",
|
||||||
|
"mask", "txs"
|
||||||
|
])
|
||||||
|
|
||||||
|
print('block saved')
|
||||||
|
|
||||||
# def setupDB():
|
# def setupDB():
|
||||||
# """Creates the database tables"""
|
# """Creates the database tables"""
|
||||||
@ -258,12 +257,12 @@ def getNamesFromBlock(height):
|
|||||||
json.dumps(nameInfo["bids"])
|
json.dumps(nameInfo["bids"])
|
||||||
))
|
))
|
||||||
|
|
||||||
query = """
|
dbSave.insert("names", queryData, column_names=[
|
||||||
INSERT INTO names (name, nameHash, state, height, lastRenewal, owner, value, highest, data, transfer, revoked, claimed, renewals, registered, expired, weak, stats, start, txs, bids)
|
"name", "nameHash", "state", "height", "lastRenewal", "owner", "value", "highest",
|
||||||
VALUES
|
"data", "transfer", "revoked", "claimed", "renewals", "registered", "expired",
|
||||||
"""
|
"weak", "stats", "start", "txs", "bids"
|
||||||
return dbSave.execute(query, queryData)
|
])
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def getNodeHeight():
|
def getNodeHeight():
|
||||||
|
@ -1,4 +1,5 @@
|
|||||||
clickhouse-driver
|
clickhouse-driver
|
||||||
|
clickhouse-connect
|
||||||
requests
|
requests
|
||||||
python-dotenv
|
python-dotenv
|
||||||
flask
|
flask
|
Loading…
Reference in New Issue
Block a user