feat: Speed up covanent using bulk

.gitignore

@@ -4,3 +4,4 @@ __pycache__/
 .env
 .vs/
 .venv/
+fireexplorer.db

Dockerfile

@@ -22,6 +22,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
 
 # Add mount point for data volume
 ENV BASE_DIR=/data
+ENV DATABASE_PATH=/data/fireexplorer.db
 VOLUME /data
 
 EXPOSE 5000

server.py

@@ -5,9 +5,12 @@ from flask import (
     send_from_directory,
     send_file,
     jsonify,
+    g,
+    request,
 )
 import os
 import requests
+import sqlite3
 from datetime import datetime
 import dotenv
 from tools import hip2, wallet_txt
@@ -16,6 +19,40 @@ dotenv.load_dotenv()
 
 app = Flask(__name__)
 
+DATABASE = os.getenv("DATABASE_PATH", "fireexplorer.db")
+
+
+def get_db():
+    db = getattr(g, "_database", None)
+    if db is None:
+        db = g._database = sqlite3.connect(DATABASE)
+        db.row_factory = sqlite3.Row
+    return db
+
+
+@app.teardown_appcontext
+def close_connection(exception):
+    db = getattr(g, "_database", None)
+    if db is not None:
+        db.close()
+
+
+def init_db():
+    with app.app_context():
+        db = get_db()
+        db.execute(
+            """
+            CREATE TABLE IF NOT EXISTS names (
+                namehash TEXT PRIMARY KEY,
+                name TEXT NOT NULL
+            )
+        """
+        )
+        db.commit()
+
+
+init_db()
+
 
 def find(name, path):
     for root, dirs, files in os.walk(path):
@@ -133,13 +170,46 @@ def catch_all(path: str):
     return render_template("404.html"), 404
 
 
+# endregion
+
+
+# region API routes
+@app.route("/api/v1/namehash/<namehash>")
+def namehash_api(namehash):
+    db = get_db()
+    cur = db.execute("SELECT * FROM names WHERE namehash = ?", (namehash,))
+    row = cur.fetchone()
+    if row is None:
+        # Get namehash from hsd.hns.au
+        req = requests.get(f"https://hsd.hns.au/api/v1/namehash/{namehash}")
+        if req.status_code == 200:
+            name = req.json().get("result")
+            if not name:
+                return jsonify({"name": "Error", "namehash": namehash})
+            # Insert into database
+            db.execute(
+                "INSERT OR REPLACE INTO names (namehash, name) VALUES (?, ?)",
+                (namehash, name),
+            )
+            db.commit()
+            return jsonify({"name": name, "namehash": namehash})
+    return jsonify(dict(row))
+
+
 @app.route("/api/v1/status")
 def api_status():
+    # Count number of names in database
+    db = get_db()
+    cur = db.execute("SELECT COUNT(*) as count FROM names")
+    row = cur.fetchone()
+    name_count = row["count"] if row else 0
+
     return jsonify(
         {
             "status": "ok",
             "service": "FireExplorer",
             "version": "1.0.0",
+            "names_cached": name_count,
         }
     )
 
@@ -176,6 +246,108 @@ def hip02(domain: str):
     )
 
 
+@app.route("/api/v1/covenant", methods=["POST"])
+def covenant_api():
+    data = request.get_json()
+
+    if isinstance(data, list):
+        covenants = data
+        results = []
+
+        # Collect all namehashes needed
+        namehashes = set()
+        for cov in covenants:
+            items = cov.get("items", [])
+            if items:
+                namehashes.add(items[0])
+
+        # Batch DB lookup
+        db = get_db()
+        known_names = {}
+        if namehashes:
+            placeholders = ",".join("?" for _ in namehashes)
+            cur = db.execute(
+                f"SELECT namehash, name FROM names WHERE namehash IN ({placeholders})",
+                list(namehashes),
+            )
+            for row in cur:
+                known_names[row["namehash"]] = row["name"]
+
+        # Identify missing namehashes
+        missing_hashes = [nh for nh in namehashes if nh not in known_names]
+
+        # Fetch missing from HSD
+        session = requests.Session()
+        for nh in missing_hashes:
+            try:
+                req = session.get(f"https://hsd.hns.au/api/v1/namehash/{nh}")
+                if req.status_code == 200:
+                    name = req.json().get("result")
+                    if name:
+                        known_names[nh] = name
+                        # Update DB
+                        db.execute(
+                            "INSERT OR REPLACE INTO names (namehash, name) VALUES (?, ?)",
+                            (nh, name),
+                        )
+            except Exception as e:
+                print(f"Error fetching namehash {nh}: {e}")
+
+        db.commit()
+
+        # Build results
+        for cov in covenants:
+            action = cov.get("action")
+            items = cov.get("items", [])
+
+            if not action:
+                results.append({"covenant": cov, "display": "Unknown"})
+                continue
+
+            display = f"{action}"
+            if items:
+                nh = items[0]
+                if nh in known_names:
+                    name = known_names[nh]
+                    display += f' <a href="/name/{name}">{name}</a>'
+
+            results.append({"covenant": cov, "display": display})
+
+        return jsonify(results)
+
+    # Get the covenant data
+    action = data.get("action")
+    items = data.get("items", [])
+
+    if not action:
+        return jsonify({"success": False, "data": data})
+
+    display = f"{action}"
+    if len(items) > 0:
+        name_hash = items[0]
+        # Lookup name from database
+        db = get_db()
+        cur = db.execute("SELECT * FROM names WHERE namehash = ?", (name_hash,))
+        row = cur.fetchone()
+        if row:
+            name = row["name"]
+            display += f' <a href="/name/{name}">{name}</a>'
+        else:
+            req = requests.get(f"https://hsd.hns.au/api/v1/namehash/{name_hash}")
+            if req.status_code == 200:
+                name = req.json().get("result")
+                if name:
+                    display += f" {name}"
+                    # Insert into database
+                    db.execute(
+                        "INSERT OR REPLACE INTO names (namehash, name) VALUES (?, ?)",
+                        (name_hash, name),
+                    )
+                    db.commit()
+
+    return jsonify({"success": True, "data": data, "display": display})
+
+
 # endregion
 
 
@@ -188,4 +360,4 @@ def not_found(e):
 
 # endregion
 if __name__ == "__main__":
-    app.run(debug=True, port=5000, host="0.0.0.0")
+    app.run(debug=True, port=5000, host="127.0.0.1")

templates/index.html

@@ -469,7 +469,7 @@
                                     <div style="display: flex; justify-content: space-between; margin-top: 0.5rem; font-size: 0.85rem; color: #b0b0b0;">
                                         <span>Height: ${coin.height.toLocaleString()}</span>
                                         <span>Coinbase: ${coin.coinbase ? 'Yes' : 'No'}</span>
-                                        <span>Covenant: ${coin.covenant.action}</span>
+                                        <span>Covenant: <span data-covenant-action="${coin.covenant.action}" data-covenant="${encodeURIComponent(JSON.stringify(coin.covenant))}">${coin.covenant.action}</span></span>
                                     </div>
                                 </div>
                             `).join('')}
@@ -521,7 +521,7 @@
                             </div>
                             <div class="info-item">
                                 <label>Covenant:</label>
-                                <span>${coin.covenant.action}</span>
+                                <span data-covenant-action="${coin.covenant.action}" data-covenant="${encodeURIComponent(JSON.stringify(coin.covenant))}">${coin.covenant.action}</span>
                             </div>
                         </div>
                     </div>
@@ -696,7 +696,7 @@
                                         recordDetails += `<br><span style="font-size: 0.9rem;">KeyTag: ${record.keyTag}, Algorithm: ${record.algorithm}, DigestType: ${record.digestType}</span><br><span class="mono" style="font-size: 0.85rem; color: #b0b0b0; word-break: break-all;">${record.digest}</span>`;
                                         break;
                                     case 'TXT':
-                                        recordDetails += `<br><span style="color: #b0b0b0;">${record.txt.map(t => `"${t}"`).join('<br>')}</span>`;
+                                        recordDetails += `<br><span style="color: #b0b0b0;">${record.txt.join('<br>')}</span>`;
                                         break;
                                     case 'GLUE4':
                                     case 'GLUE6':
@@ -779,7 +779,7 @@
                                             recordDetails += `<br><span style="font-size: 0.9rem;">KeyTag: ${record.keyTag}, Algorithm: ${record.algorithm}</span>`;
                                             break;
                                         case 'TXT':
-                                            recordDetails += `<br><span style="color: #b0b0b0;">${record.txt.map(t => `"${t}"`).join('<br>')}</span>`;
+                                            recordDetails += `<br><span style="color: #b0b0b0;">${record.txt.join('<br>')}</span>`;
                                             break;
                                         case 'GLUE4':
                                         case 'GLUE6':
@@ -861,7 +861,7 @@
                                                 <span class="tx-io-value">${input.coin ? formatValue(input.coin.value) : 'Unknown'}</span>
                                             </div>
                                             <div class="tx-io-address">${input.coin ? input.coin.address : (input.address || 'Unknown')}</div>
-                                            ${input.coin && input.coin.covenant.action !== 'NONE' ? `<div class="tx-covenant">Covenant: ${input.coin.covenant.action}</div>` : ''}
+                                            ${input.coin && input.coin.covenant.action !== 'NONE' ? `<div class="tx-covenant" data-covenant-action="${input.coin.covenant.action}" data-covenant="${encodeURIComponent(JSON.stringify(input.coin.covenant))}">Covenant: ${input.coin.covenant.action}</div>` : ''}
                                         </div>
                                     `;
                                 }
@@ -879,7 +879,7 @@
                                         <span class="tx-io-value">${formatValue(output.value)}</span>
                                     </div>
                                     <div class="tx-io-address">${output.address}</div>
-                                    ${output.covenant.action !== 'NONE' ? `<div class="tx-covenant">Covenant: ${output.covenant.action}</div>` : ''}
+                                    ${output.covenant.action !== 'NONE' ? `<div class="tx-covenant" data-covenant-action="${output.covenant.action}" data-covenant="${encodeURIComponent(JSON.stringify(output.covenant))}">Covenant: ${output.covenant.action}</div>` : ''}
                                 </div>
                             `).join('')}
                         </div>
@@ -919,6 +919,7 @@
                 </div>
             `;
             document.body.appendChild(modal);
+            if (!data.error) updateCovenants();
         }
 
         // Display helper
@@ -940,6 +941,74 @@
             }
         }
 
+        // Update covenant information from API
+        async function updateCovenants() {
+            const elements = document.querySelectorAll('[data-covenant-action]');
+            const covenantsToFetch = [];
+            const elementMap = new Map(); // Map JSON string -> Array of elements
+
+            for (const el of elements) {
+                const action = el.dataset.covenantAction;
+                if (action === 'NONE') continue;
+
+                // Skip if already updated
+                if (el.dataset.covenantUpdated) continue;
+
+                // Get full covenant data
+                let covenantData = null;
+                if (el.dataset.covenant) {
+                    try {
+                        covenantData = JSON.parse(decodeURIComponent(el.dataset.covenant));
+                    } catch (e) {
+                        console.error('Failed to parse covenant data:', e);
+                    }
+                }
+
+                if (!covenantData) continue;
+
+                const key = JSON.stringify(covenantData);
+                if (!elementMap.has(key)) {
+                    elementMap.set(key, []);
+                    covenantsToFetch.push(covenantData);
+                }
+                elementMap.get(key).push(el);
+            }
+
+            if (covenantsToFetch.length === 0) return;
+
+            try {
+                const res = await fetch(`/api/v1/covenant`, {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json'
+                    },
+                    body: JSON.stringify(covenantsToFetch)
+                });
+
+                if (res.ok) {
+                    const results = await res.json();
+
+                    for (const result of results) {
+                        const key = JSON.stringify(result.covenant);
+                        const els = elementMap.get(key);
+
+                        if (els) {
+                            for (const el of els) {
+                                if (el.classList.contains('tx-covenant')) {
+                                    el.innerHTML = `Covenant: ${result.display}`;
+                                } else {
+                                    el.innerHTML = result.display;
+                                }
+                                el.dataset.covenantUpdated = "true";
+                            }
+                        }
+                    }
+                }
+            } catch (e) {
+                console.error('Failed to fetch covenant info:', e);
+            }
+        }
+
         // Show loading animation
         function showLoading(elementId) {
             const element = document.getElementById(elementId);
@@ -1031,6 +1100,7 @@
                 resultElement.innerHTML = `<div class="error">Error: ${data.error}</div>`;
             } else {
                 resultElement.innerHTML = formatTransactionData(data);
+                updateCovenants();
             }
         }
 
@@ -1106,6 +1176,7 @@
                 resultElement.innerHTML = `<div class="error">Error: ${data.error}</div>`;
             } else {
                 resultElement.innerHTML = formatAddressCoins(data);
+                updateCovenants();
             }
         }
 
@@ -1178,18 +1249,24 @@
             }
             
             showLoading('name-result');
-            const data = await apiCall(`namehash/${nameHash}`);
             
-            // Check if result is valid and redirect to name page
+            try {
+                const response = await fetch(`/api/v1/namehash/${nameHash}`);
+                const data = await response.json();
+                
                 const resultElement = document.getElementById('name-result');
                 if (data.error) {
-                resultElement.innerHTML = `<div class="error">Error: ${data.error.message ? data.error.message : "Failed to lookup hash"}</div>`;
+                    resultElement.innerHTML = `<div class="error">Error: ${data.error}</div>`;
-            } else if (data.result && typeof data.result === 'string') {
+                } else if (data.name) {
                     // Valid name found, redirect to name page
-                window.location.href = `/name/${data.result}`;
+                    window.location.href = `/name/${data.name}`;
                 } else {
                     resultElement.innerHTML = `<div class="error">No name found for this hash</div>`;
                 }
+            } catch (e) {
+                const resultElement = document.getElementById('name-result');
+                resultElement.innerHTML = `<div class="error">Error: ${e.message}</div>`;
+            }
         }
 
         // Load status when page loads