From a74943dec36e29f2579e08aa244d455d152b6714 Mon Sep 17 00:00:00 2001
From: Nathan Woodburn <github@nathan.woodburn.au>
Date: Sat, 12 Aug 2023 22:39:36 +1000
Subject: [PATCH] bot: Made curl convert to md

---
 bot.py           | 37 ++++++++++++++++++++++++++++++++++---
 requirements.txt |  3 ++-
 2 files changed, 36 insertions(+), 4 deletions(-)

diff --git a/bot.py b/bot.py
index 86d06ad..8bf96bb 100644
--- a/bot.py
+++ b/bot.py
@@ -4,6 +4,7 @@ import discord
 from discord import app_commands
 import requests
 import dns.resolver
+import markdownify
 
 load_dotenv()
 TOKEN = os.getenv('DISCORD_TOKEN')
@@ -90,11 +91,41 @@ async def curl(ctx, url: str):
         proxyURL = "https://proxy.hnsproxy.au"
         response = requests.get(url, proxies={"http": proxyURL, "https": proxyURL},verify=False)
         output = response.text
-        if (len(output) > 1900):
-            output = output[:1900]
-        await ctx.response.send_message(f"Response for {url}:\n{output}")
+        # Get BODY only
+        output = output.split("<body")[1]
+        output = output.split("</body>")[0]
+        output = output.split(">", 1)[1]
+        # Replace any relative links with absolute links
+        output = output.replace('href="/', 'href="' + url + '/')
+        parsed = markdownify.markdownify(output, heading_style="ATX")
+        # Delete any empty lines
+        parsed = "\n".join([s for s in parsed.splitlines() if s.strip()])
+
+        output = response.text
+        # Get title
+        if (output.find("<title>") != -1):
+            title = output.split("<title>")[1]
+            title = title.split("</title>")[0]
+        else:
+            title = url
+        if (len(parsed) > 4096):
+            parsed = parsed[:4096]
+            # Delete any incomplete lines
+            parsed = "\n".join(parsed.splitlines()[:-1])
+            parsed = parsed + "\n..."
+
+        # if url is a tld only replace it with the url https://hns.au (due to Discord not allowing tld only links)
+        if (url.find(".") == -1):
+            url = "https://hns.au"
+        # Create an embed
+        embed = discord.Embed(title=title, url=url, description=parsed)
+        embed.set_footer(text="Parsed by HNSProxy",icon_url="https://hns.au/assets/img/favicon.png")
+        embed.timestamp = discord.utils.utcnow()
+        await ctx.response.send_message(embed=embed)
     except requests.exceptions.RequestException as e:
         await ctx.response.send_message(f"An error occurred: {e}")
+    except Exception as e:
+        await ctx.response.send_message(f"An error occurred: {e}")
     
 @tree.command(name="invite", description="Invite me to your server")
 async def invite(ctx):
diff --git a/requirements.txt b/requirements.txt
index 60d22f1..3b3b5a1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,5 @@
 discord.py
 python-dotenv
 requests
-dnspython
\ No newline at end of file
+dnspython
+markdownify
\ No newline at end of file