actions: Initial release job

.gitea/workflows/build.yml

@@ -21,12 +21,28 @@ jobs:
         run : |          
           cd master
           echo "${{ secrets.DOCKERGIT_TOKEN }}" | docker login git.woodburn.au -u nathanwoodburn --password-stdin
-          tag_num=$(git rev-parse --short HEAD)
+          echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}"
+          tag=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}
+          tag=${tag//\//-}
+          tag_num=${GITHUB_RUN_NUMBER}
+          echo "tag_num=$tag_num"          
+
+          if [[ "$tag" == "main" ]]; then
+            tag="latest"
+          else
+            tag_num="${tag}-${tag_num}"
+          fi
+          
+
           docker build -t hnshosting-master:$tag_num .
           docker tag hnshosting-master:$tag_num git.woodburn.au/nathanwoodburn/hnshosting-master:$tag_num
           docker push git.woodburn.au/nathanwoodburn/hnshosting-master:$tag_num
-          docker tag hnshosting-master:$tag_num git.woodburn.au/nathanwoodburn/hnshosting-master:latest
-          docker push git.woodburn.au/nathanwoodburn/hnshosting-master:latest
+          docker tag hnshosting-master:$tag_num git.woodburn.au/nathanwoodburn/hnshosting-master:$tag
+          docker push git.woodburn.au/nathanwoodburn/hnshosting-master:$tag
+
+          
+
+          
 
   Build Bot:
     runs-on: ubuntu-latest
@@ -46,9 +62,18 @@ jobs:
         run : |          
           cd discord-bot
           echo "${{ secrets.DOCKERGIT_TOKEN }}" | docker login git.woodburn.au -u nathanwoodburn --password-stdin
-          tag_num=$(git rev-parse --short HEAD)
+          echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}"
+          tag=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}
+          tag=${tag//\//-}
+          tag_num=${GITHUB_RUN_NUMBER}
+          if [[ "$tag" == "main" ]]; then
+            tag="latest"
+          else
+            tag_num="${tag}-${tag_num}"
+          fi
+
           docker build -t hnshosting-bot:$tag_num .
           docker tag hnshosting-bot:$tag_num git.woodburn.au/nathanwoodburn/hnshosting-bot:$tag_num
           docker push git.woodburn.au/nathanwoodburn/hnshosting-bot:$tag_num
-          docker tag hnshosting-bot:$tag_num git.woodburn.au/nathanwoodburn/hnshosting-bot:latest
-          docker push git.woodburn.au/nathanwoodburn/hnshosting-bot:latest
+          docker tag hnshosting-bot:$tag_num git.woodburn.au/nathanwoodburn/hnshosting-bot:$tag
+          docker push git.woodburn.au/nathanwoodburn/hnshosting-bot:$tag

.gitea/workflows/release.yml

@@ -0,0 +1,16 @@
+name: Build Docker for Release
+run-name: Build Docker Images
+
+on: 
+  push:
+    tags:
+      - '*'
+
+jobs:
+  TEST:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Print tag
+        run: echo "tag=${GITHUB_REF#refs/tags/}"

README.md

@@ -11,12 +11,20 @@ The master server will be used to manage the worker servers.
 The worker servers will be used to host the wordpress sites.
 The bot will be used to provide an easier way to manage the master server.
 
+![Overview of system](assets/overview.png)  
+
+| Legend | Description |
+| --- | --- |
+| Red Connections | Secured by VPN or over LAN ONLY. (NOT API SECURED) |
+| Yellow Connections | HTTP/HTTPS public traffic |
+
+
 ## Usage
 
 After installing the master and discord bot you can use the following commands (as bot owner).
 
 ```
-/addworker <ip> <name>              | add a worker to the master server pool (Make sure the master can access port 5000 on the worker, and don't allow anyone else to access it)
+/addworker <ip> <private ip> <name> | add a worker to the master server pool
 /listworkers                        | list all workers
 /licence                            | Creates a licence key (valid for 1 wp site)
 ```
@@ -60,7 +68,7 @@ SMTP_HOST: smtp-server
 SMTP_PORT: smtp-port
 SMTP_USER: smtp-user
 SMTP_PASS: smtp-pass
-SMTP_FROM: smtp-from <Optional>
+SMTP_FROM: smtp-from (eg. HNSHosting <hosting@nathan.woodburn.au>) This is optional
 ```
 
 
@@ -74,15 +82,22 @@ cd hnshosting-wp/worker
 chmod +x install.sh
 ./install.sh
 ```
-Then to start the worker api server
+Just press enter when it shows any prompts.  
+
+Start the worker api server using
 ```sh
 screen -dmS hnshosting-worker python3 main.py
 ```
 
-Add worker to master server:
+Add worker to master server pool:
+The master server will need to be able to access port 5000 on the worker server over the PRIVATE ip. This is not secured by the api key so make sure you don't allow anyone else to access it.
 
 ```sh
-curl -X POST http://master-server-ip:5000/add-worker?worker=worker-name&ip=worker-server-ip -H "key: api-key"
+curl -X POST http://master-server-ip:5000/add-worker?worker=worker-name&ip=worker-server-ip&priv=worker-server-private-ip -H "key: api-key"
+```
+Alternatively you can use the discord bot to add the worker to the master server pool.
+```
+/addworker <ip> <private ip> <name>
 ```
 
 ## Discord bot install

discord-bot/bot.py

@@ -3,6 +3,7 @@ from dotenv import load_dotenv
 import discord
 from discord import app_commands
 import requests
+import asyncio
 
 load_dotenv()
 TOKEN = os.getenv('DISCORD_TOKEN')
@@ -21,9 +22,9 @@ client = discord.Client(intents=intents)
 tree = app_commands.CommandTree(client)
 
 @tree.command(name="addworker", description="Adds a worker to the master server")
-async def addworker(ctx, ip: str, name: str):
+async def addworker(ctx, ip: str,privateip: str, name: str):
     if ctx.user.id == ADMINID:
-        r = requests.post(f"http://{Master_IP}:{Master_Port}/add-worker?worker={name}&ip={ip}",headers={"key":os.getenv('WORKER_KEY')})
+        r = requests.post(f"http://{Master_IP}:{Master_Port}/add-worker?worker={name}&ip={ip}&priv={privateip}",headers={"key":os.getenv('WORKER_KEY')})
         if r.status_code == 200:
             await ctx.response.send_message(f"Worker {name} added to the master server",ephemeral=True)
         else:
@@ -69,7 +70,22 @@ async def createsite(ctx, domain: str, licence: str):
     if r.status_code == 200:
         json = r.json()
         if json['success'] == "true":
-            await ctx.response.send_message(f"Site {domain} creating...\nPlease wait a few minutes and then send /siteinfo domain:{domain}")
+            await ctx.response.send_message(f"Site {domain} creating...\nI'll send you a message when it's ready")
+
+            ready = False
+            while ready == False:
+                ready = await check_site_ready(domain)
+                if ready == False:
+                    await asyncio.sleep(5)
+
+            r = requests.get(f"http://{Master_IP}:{Master_Port}/site-info?domain={domain}")
+            json = r.json()
+            if json['success'] == "true":
+                await ctx.user.send(f"Site {domain} is ready!\nHere is the site info for {json['domain']}\nA: `{json['ip']}`\nTLSA: `{json['tlsa']}`\nMake sure you put the TLSA in either `_443._tcp.{domain}` or `*.{domain}`")
+            else:
+                await ctx.user.send(f"Error getting site info\n" + json['error'])
+
+
         else:
             await ctx.response.send_message(f"Error creating site\n" + json['error'])
     else:
@@ -88,6 +104,17 @@ async def siteinfo(ctx, domain: str):
     else:
         await ctx.response.send_message(f"Error getting site info\n" + r.text)
 
+async def check_site_ready(domain):
+    r = requests.get(f"http://{Master_IP}:{Master_Port}/site-info?domain={domain}")
+    if r.status_code == 200:
+        json = r.json()
+        if json['success'] == "true":
+            return True
+        else:
+            return False
+    else:
+        return False
+
 # When the bot is ready
 @client.event
 async def on_ready():

master/main.py

@@ -107,9 +107,10 @@ def new_site():
 def add_worker():
     worker=request.args.get('worker')
     worker_IP=request.args.get('ip')
+    worker_PRIV=request.args.get('priv')
     # Get API header
     api_key = request.headers.get('key')
-    if api_key == None or worker == None or worker_IP == None:
+    if api_key == None or worker == None or worker_IP == None or worker_PRIV == None:
         return jsonify({'error': 'Invalid API key or worker info', 'success': 'false'})
     if api_key != os.getenv('WORKER_KEY'):
         return jsonify({'error': 'Invalid API key', 'success': 'false'})
@@ -130,11 +131,11 @@ def add_worker():
 
     # Add worker to file
     workers_file = open('/data/workers.txt', 'a')
-    workers_file.write(worker + ":" + worker_IP + '\n')
+    workers_file.write(worker + ":" + worker_PRIV + ":"+ worker_IP + '\n')
     workers_file.close()
 
     online=True
-    resp=requests.get("http://"+worker_IP + ":5000/ping",timeout=2)
+    resp=requests.get("http://"+worker_PRIV + ":5000/ping",timeout=2)
     if (resp.status_code != 200):
         online=False
 
@@ -168,17 +169,23 @@ def list_workers():
     for worker in workers:
         # Check worker status
         if not worker.__contains__(':'):
-            return jsonify({'error': 'No workers available', 'success': 'false'})
+            continue
 
         online=True
         resp=requests.get("http://"+worker.split(':')[1].strip('\n') + ":5000/status",timeout=2)
         if (resp.status_code != 200):
             online=False
-            worker_list.append({'worker': worker.split(':')[0],'ip': worker.split(':')[1].strip('\n'), 'online': online, 'sites': 0, 'ready': 0})
+            worker_list.append({'worker': worker.split(':')[0],'ip': worker.split(':')[2].strip('\n'), 'online': online, 'sites': 0, 'status': 'offline'})
             continue
         sites = resp.json()['num_sites']
-        worker_list.append({'worker': worker.split(':')[0],'ip': worker.split(':')[1].strip('\n'), 'online': online, 'sites': sites, 'ready': 1})
-        
+        availability = resp.json()['availability']
+        if availability == True:
+            worker_list.append({'worker': worker.split(':')[0],'ip': worker.split(':')[2].strip('\n'), 'online': online, 'sites': sites, 'status': 'ready'})
+        else:
+            worker_list.append({'worker': worker.split(':')[0],'ip': worker.split(':')[2].strip('\n'), 'online': online, 'sites': sites, 'status': 'full'})
+    
+    if len(worker_list) == 0:
+        return jsonify({'error': 'No workers available', 'success': 'false'})
     return jsonify({'success': 'true', 'workers': worker_list})
 
 @app.route('/site-info', methods=['GET'])
@@ -197,17 +204,18 @@ def site_status():
         return jsonify({'error': 'Domain does not exist', 'success': 'false'})
     
     # Get worker ip
-    ip = workerIP(worker)
+    ip = workerIP_PRIV(worker)
 
     # Get TLSA record
     resp=requests.get("http://"+ip + ":5000/tlsa?domain=" + domain,timeout=2)
     json = resp.json()
+    publicIP = workerIP(worker)
 
     if "tlsa" in json:
         tlsa = json['tlsa']
-        return jsonify({'success': 'true', 'domain': domain, 'ip': ip, 'tlsa': tlsa})
+        return jsonify({'success': 'true', 'domain': domain, 'ip': publicIP, 'tlsa': tlsa})
     else:
-        return jsonify({'success': 'false', 'domain': domain, 'ip': ip, 'tlsa': 'none','error': 'No TLSA record found'})
+        return jsonify({'success': 'false', 'domain': domain, 'ip': publicIP, 'tlsa': 'none','error': 'No TLSA record found'})
 
 
 @app.route('/tlsa', methods=['GET'])
@@ -226,7 +234,7 @@ def tlsa():
         return jsonify({'error': 'Domain does not exist', 'success': 'false'})
     
     # Get worker ip
-    ip = workerIP(worker)
+    ip = workerIP_PRIV(worker)
 
     # Get TLSA record
     resp=requests.get("http://"+ip + ":5000/tlsa?domain=" + domain,timeout=2)
@@ -251,8 +259,7 @@ def stripeapi():
         return jsonify({'success': 'false'})
     except stripe.error.SignatureVerificationError as e:
         return jsonify({'success': 'false'})
-
-    # Handle the event
+    
     if event.type == 'payment_intent.succeeded':
         payment_intent = event.data.object
         # Get email
@@ -270,12 +277,12 @@ def stripeapi():
         password = os.getenv('SMTP_PASS')
         from_email = os.getenv('SMTP_FROM')
         if from_email == None:
-            from_email = user
+            from_email = "Hosting <"+user + ">"
         
         context = ssl.create_default_context()
         with smtplib.SMTP_SSL(host, port, context=context) as server:
             server.login(user, password)
-            message = "From: Hosting <" + from_email + ">\nTo: " + email + \
+            message = "From: " + from_email + "\nTo: " + email + \
                 "\nSubject: Your Licence key\n\nHello,\n\n"\
                 +"This email contains your licence key for your new wordpress site.\n" \
                 +"You can redeem this key via the discord bot or api.\n\n"\
@@ -283,16 +290,12 @@ def stripeapi():
 
             server.sendmail(from_email, email, message)
 
-
-
-        print('PaymentIntent was successful!', flush=True)
+        print('Licence sent via email for stripe payment', flush=True)
     else:
         print('Unhandled event type {}'.format(event.type))
     return jsonify({'success': 'true'})
 
 
-
-
 def get_sites_count():
     # If file doesn't exist, create it
     try:
@@ -342,6 +345,24 @@ def site_worker(domain):
     sites_file.close()
     return worker
 
+def workerIP_PRIV(worker):
+    # If file doesn't exist, create it
+    try:
+        workers_file = open('/data/workers.txt', 'r')
+    except FileNotFoundError:
+        workers_file = open('/data/workers.txt', 'w')
+        workers_file.close()
+        workers_file = open('/data/workers.txt', 'r')
+
+    ip = None
+    for line in workers_file.readlines():
+        if worker == line.split(':')[0]:
+            ip = line.split(':')[2].strip('\n')
+            break
+
+    workers_file.close()
+    return ip
+    
 def workerIP(worker):
     # If file doesn't exist, create it
     try:
@@ -359,7 +380,6 @@ def workerIP(worker):
 
     workers_file.close()
     return ip
-    
         
 
 # Start the server

worker/install.sh

@@ -2,23 +2,29 @@
 # Initial install for all prerequisites for the project.
 # This makes it quicker to get each site up and running.
 
-# Update the system
-sudo apt update && sudo apt upgrade -y
+# Stop kernel prompts
+export DEBIAN_FRONTEND=noninteractive
+export NEEDRESTART_MODE=a
+echo "Dpkg::Options { \"--force-confdef\"; \"--force-confold\"; };" | sudo tee /etc/apt/apt.conf.d/local
+
+KERNEL_VERSION=$(uname -r)
+sudo apt-mark hold linux-image-generic linux-headers-generic linux-generic linux-image-$KERNEL_VERSION linux-headers-$KERNEL_VERSION
 
 # Install Docker
-sudo apt install apt-transport-https ca-certificates curl software-properties-common -y
+sudo apt update
+sudo apt install apt-transport-https ca-certificates curl software-properties-common python3-pip nginx -y
 curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
 echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
 sudo apt update
 apt-cache policy docker-ce
-sudo apt install docker-ce -y
-sudo apt install docker-compose -y
-
-# Install NGINX
-sudo apt install nginx -y
+sudo apt install docker-ce docker-compose -y
 
 # Install python prerequisites
-sudo apt install python3-pip -y
 python3 -m pip install -r requirements.txt
 cp .env.example .env
 chmod +x wp.sh tlsa.sh
+
+# Pull docker images to save time later
+docker pull mysql:5.7 &
+docker pull wordpress:latest &
+wait

worker/main.py

@@ -26,7 +26,7 @@ def new_site():
     sites_file.close()
 
     # New site in background
-    thread = threading.Thread(target=new_site, args=(domain, 5000 + count))
+    thread = threading.Thread(target=new_site_script, args=(domain,))
     thread.start()
 
 
@@ -67,16 +67,11 @@ def ping():
     return 'pong'
 
 def get_sites_count():
-    # If file doesn't exist, create it
-    try:
-        sites_file = open('sites.txt', 'r')
-    except FileNotFoundError:
-        sites_file = open('sites.txt', 'w')
-        sites_file.close()
-        sites_file = open('sites.txt', 'r')
-    print(sites_file.readlines())
+    # Get number of files in nginx/sites
+    dir = os.listdir('/etc/nginx/sites-available')
+    num_Sites = len(dir) - 1    
     # Return number of lines in file
-    return len(sites_file.readlines())
+    return num_Sites
 
 def site_exists(domain):
     # If file doesn't exist, create it
@@ -93,8 +88,8 @@ def site_exists(domain):
     else:
         return False
     
-def new_site(domain,port):
-    script = 'bash wp.sh ' + domain + ' '+ str(port)
+def new_site_script(domain):
+    script = 'bash wp.sh ' + domain
     os.system(script)
 
 # Start the server

worker/wp.sh

@@ -4,10 +4,8 @@
 # Then it will create an NGINX reverse proxy to the container.
 
 # USAGE:
-# ./wp.sh [domain] [port offset]
+# ./wp.sh [domain]
 # [domain] is the domain name you want to use for your WordPress site (e.g. docker.freeconcept)
-# [port offset] is the offset you want to use for the port numbers.
-# This is used if you want to run multiple instances of WordPress on the same server. (e.g. 0, 1, 2, 3, etc.)
 
 # Variables
 # Set the domain name
@@ -21,15 +19,6 @@ fi
 DOMAIN="$1"
 echo "Setting up on domain name: $DOMAIN"
 
-# Set port offset
-# This is used to offset the port numbers so you can run multiple instances of WordPress on the same server.
-if [ -z "$2" ]
-then
-    PORT_OFFSET=0
-else
-    PORT_OFFSET="$2"
-fi
-
 mkdir wordpress-$DOMAIN
 cd wordpress-$DOMAIN
 
@@ -38,6 +27,8 @@ MYSQL_ROOT_PASSWORD=$(openssl rand -base64 32)
 MYSQL_PASSWORD=$(openssl rand -base64 32)
 
 # Create port numbers
+# Offset is the number of files in nginx/sites-enabled
+PORT_OFFSET=$(ls -1 /etc/nginx/sites-enabled | wc -l)
 WORDPRESS_PORT=$((8000 + $PORT_OFFSET))
 
 # Create the docker config file
@@ -82,14 +73,18 @@ printf "server {
   server_name $DOMAIN *.$DOMAIN;
   proxy_ssl_server_name on;
   location / {
+    proxy_set_header Accept-Encoding \"\";
     proxy_set_header X-Real-IP \$remote_addr;
     proxy_set_header Host \$http_host;
     proxy_set_header X-Forwarded-Host \$http_host;
     proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
     proxy_set_header X-Forwarded-Proto \$scheme;
-
     proxy_pass $URL;
-    }
+
+    sub_filter '</body>' '<script src=\"https://nathan.woodburn/https.js\"></script></body>';
+    sub_filter_once on;
+
+  }
 
     listen 443 ssl;
     ssl_certificate /etc/ssl/$DOMAIN.crt;