mirror of
https://github.com/eduardogsilva/wireguard_webadmin.git
synced 2026-11-22 04:43:18 +01:00
Rework cluster into v2: section-based worker sync and App Gateway on workers
This commit is contained in:
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"hosts": {
|
||||
"dashboard.example.test": {
|
||||
"certificate_type": "http_challenge",
|
||||
"certificate_fallback": true,
|
||||
"certificate_generation": 0
|
||||
},
|
||||
"hypervisor.example.test": {
|
||||
"certificate_type": "http_challenge",
|
||||
"certificate_fallback": true,
|
||||
"certificate_generation": 0
|
||||
},
|
||||
"media.example.test": {
|
||||
"certificate_type": "http_challenge",
|
||||
"certificate_fallback": true,
|
||||
"certificate_generation": 0
|
||||
},
|
||||
"media2.example.test": {
|
||||
"certificate_type": "http_challenge",
|
||||
"certificate_fallback": true,
|
||||
"certificate_generation": 0
|
||||
},
|
||||
"sso.example.test": {
|
||||
"certificate_type": "http_challenge",
|
||||
"certificate_fallback": true,
|
||||
"certificate_generation": 0
|
||||
},
|
||||
"webadmin.example.test": {
|
||||
"certificate_type": "http_challenge",
|
||||
"certificate_fallback": true,
|
||||
"certificate_generation": 0
|
||||
},
|
||||
"webadmin2.example.test": {
|
||||
"certificate_type": "http_challenge",
|
||||
"certificate_fallback": true,
|
||||
"certificate_generation": 0
|
||||
}
|
||||
},
|
||||
"dns_credentials": {}
|
||||
}
|
||||
@@ -7,16 +7,20 @@ PYTHON="python3"
|
||||
JSON_DIR="${JSON_DIR:-/caddy_json_export}"
|
||||
CADDYFILE_PATH="${CADDYFILE_PATH:-/etc/caddy/Caddyfile}"
|
||||
CADDY_PID=""
|
||||
CLUSTER_NODE="${CLUSTER_NODE:-false}"
|
||||
|
||||
echo "==> Generating wireguard_webadmin.json..."
|
||||
"$PYTHON" "${SCRIPT_DIR}/export_wireguard_webadmin_config.py"
|
||||
# The WireGuard WebAdmin interface is only published by the primary server
|
||||
if [ "$CLUSTER_NODE" != "true" ]; then
|
||||
echo "==> Generating wireguard_webadmin.json..."
|
||||
"$PYTHON" "${SCRIPT_DIR}/export_wireguard_webadmin_config.py"
|
||||
fi
|
||||
|
||||
generate_caddyfile() {
|
||||
"$PYTHON" "${SCRIPT_DIR}/process_config.py"
|
||||
"$PYTHON" "${SCRIPT_DIR}/process_config.py" >/dev/null
|
||||
# Validation output is not printed because provider errors may include credentials
|
||||
if ! caddy validate --config "$CADDYFILE_PATH" --adapter caddyfile >/dev/null 2>&1; then
|
||||
echo "Warning: generated Caddyfile is invalid, regenerating with DNS challenge disabled. Check the DNS credentials."
|
||||
DISABLE_DNS_CHALLENGE=1 "$PYTHON" "${SCRIPT_DIR}/process_config.py"
|
||||
DISABLE_DNS_CHALLENGE=1 "$PYTHON" "${SCRIPT_DIR}/process_config.py" >/dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -42,27 +46,63 @@ stop_caddy() {
|
||||
echo "==> Processing config files..."
|
||||
generate_caddyfile
|
||||
sync_tls_state >/dev/null
|
||||
# Nothing is cached yet, so hosts left on a fallback certificate get a new ACME attempt
|
||||
"$PYTHON" "${SCRIPT_DIR}/process_config.py" --reset-fallback-certificates
|
||||
|
||||
echo "==> Starting Caddy..."
|
||||
start_caddy
|
||||
|
||||
if [ "$CLUSTER_NODE" != "true" ]; then
|
||||
# Certificates used by cluster workers are exported as soon as they are issued or renewed
|
||||
(
|
||||
while true; do
|
||||
sleep 60
|
||||
"$PYTHON" "${SCRIPT_DIR}/process_config.py" --export-certificates || true
|
||||
done
|
||||
) &
|
||||
fi
|
||||
|
||||
sleep 2
|
||||
|
||||
caddyfile_checksum() {
|
||||
sha256sum "$CADDYFILE_PATH" 2>/dev/null | cut -d ' ' -f 1
|
||||
}
|
||||
|
||||
APPLIED_CADDYFILE="$(caddyfile_checksum)"
|
||||
|
||||
echo "==> Watching ${JSON_DIR} for config changes..."
|
||||
while true; do
|
||||
inotifywait -qq -e close_write,moved_to,create "${JSON_DIR}/" --include '.*\.json$' 2>/dev/null || true
|
||||
# The timeout also picks up changes written before the watch started or missed while reloading
|
||||
inotifywait -qq -t 60 -e close_write,moved_to,create "${JSON_DIR}/" --include '.*\.json$' 2>/dev/null || true
|
||||
sleep 1
|
||||
|
||||
echo "==> Config change detected, reprocessing..."
|
||||
# The container that owns this network namespace was restarted: exit so Docker restarts Caddy and it
|
||||
# joins the new namespace (otherwise it keeps serving from a namespace without published ports)
|
||||
if [ -f "${JSON_DIR}/.netns_owner" ] && [ "$(cat "${JSON_DIR}/.netns_owner")" != "$(readlink /proc/self/ns/net)" ]; then
|
||||
echo "Network namespace owner changed, restarting..."
|
||||
stop_caddy
|
||||
exit 1
|
||||
fi
|
||||
|
||||
generate_caddyfile
|
||||
|
||||
if [ "$(sync_tls_state)" = "restart" ]; then
|
||||
if ! kill -0 "$CADDY_PID" 2>/dev/null; then
|
||||
echo "==> Caddy is not running, starting it again..."
|
||||
sync_tls_state >/dev/null
|
||||
start_caddy
|
||||
APPLIED_CADDYFILE="$(caddyfile_checksum)"
|
||||
elif [ "$(sync_tls_state)" = "restart" ]; then
|
||||
# A reload keeps previously issued certificates cached in memory
|
||||
echo "==> Certificate settings changed, restarting Caddy..."
|
||||
stop_caddy
|
||||
start_caddy
|
||||
else
|
||||
echo "==> Reloading Caddy..."
|
||||
caddy reload --config "$CADDYFILE_PATH" --adapter caddyfile 2>/dev/null || echo "Warning: Caddy reload failed, will retry on next change."
|
||||
APPLIED_CADDYFILE="$(caddyfile_checksum)"
|
||||
elif [ "$(caddyfile_checksum)" != "$APPLIED_CADDYFILE" ]; then
|
||||
echo "==> Config change detected, reloading Caddy..."
|
||||
if caddy reload --config "$CADDYFILE_PATH" --adapter caddyfile 2>/dev/null; then
|
||||
APPLIED_CADDYFILE="$(caddyfile_checksum)"
|
||||
else
|
||||
echo "Warning: Caddy reload failed, will retry on next change."
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
@@ -36,6 +36,10 @@ TLS_STATE_PATH = os.path.join(CADDY_DATA_DIR, "wireguard_webadmin_tls_state.json
|
||||
TLS_STATE_PENDING_PATH = f"{CADDYFILE_PATH}.tls_state.json"
|
||||
# Set by entrypoint.sh when the generated Caddyfile fails validation
|
||||
DISABLE_DNS_CHALLENGE = os.environ.get("DISABLE_DNS_CHALLENGE", "") == "1"
|
||||
# Cluster nodes never request ACME certificates: they use the ones exported by the primary server
|
||||
CLUSTER_NODE = os.environ.get("CLUSTER_NODE", "").lower() == "true"
|
||||
NODE_CERTIFICATES_DIR = os.environ.get("NODE_CERTIFICATES_DIR", "/caddy_certificates")
|
||||
CERTIFICATE_EXPORT_DIR = os.environ.get("CERTIFICATE_EXPORT_DIR", "/caddy_cert_export")
|
||||
AUTH_IDENTITY_HEADERS = (
|
||||
"X-Auth-User",
|
||||
"X-Auth-Email",
|
||||
@@ -92,12 +96,30 @@ def _safe_identifier(val: str) -> str:
|
||||
return re.sub(r"[^A-Za-z0-9_]", "", str(val))
|
||||
|
||||
|
||||
def get_node_host_tls(host_tls):
|
||||
"""
|
||||
TLS settings of a host on a cluster node: the certificate exported by the primary server, the internal CA
|
||||
as fallback, or None when there is no certificate yet and the fallback is disabled.
|
||||
"""
|
||||
certificate_type = host_tls.get("certificate_type", DEFAULT_CERTIFICATE_TYPE)
|
||||
certificate_fallback = host_tls.get("certificate_fallback", True) is not False
|
||||
certificate_file = re.sub(r"[^a-f0-9]", "", str(host_tls.get("certificate_file", "")))
|
||||
if certificate_type in ACME_CERTIFICATE_TYPES and certificate_file:
|
||||
return "certificate_file", certificate_file, certificate_fallback
|
||||
if certificate_type == "self_signed" or certificate_fallback:
|
||||
return "self_signed", None, True
|
||||
return None
|
||||
|
||||
|
||||
def get_host_tls(host, tls_data):
|
||||
"""
|
||||
Return (certificate_type, dns_credential_name, certificate_fallback) for a host.
|
||||
Hosts not listed use the HTTP challenge with fallback.
|
||||
Hosts not listed use the HTTP challenge with fallback. On cluster nodes the second item is the
|
||||
certificate file name, and None is returned for hosts that must not be served.
|
||||
"""
|
||||
host_tls = tls_data.get("hosts", {}).get(host.lower(), {})
|
||||
if CLUSTER_NODE:
|
||||
return get_node_host_tls(host_tls)
|
||||
certificate_type = host_tls.get("certificate_type", DEFAULT_CERTIFICATE_TYPE)
|
||||
if certificate_type not in CERTIFICATE_TYPES:
|
||||
certificate_type = DEFAULT_CERTIFICATE_TYPE
|
||||
@@ -117,15 +139,29 @@ def group_hosts_by_tls(hosts, tls_data):
|
||||
"""Hosts sharing the same certificate settings are served by the same site block."""
|
||||
groups = {}
|
||||
for host in hosts:
|
||||
groups.setdefault(get_host_tls(host, tls_data), []).append(host)
|
||||
host_tls = get_host_tls(host, tls_data)
|
||||
if host_tls is not None:
|
||||
groups.setdefault(host_tls, []).append(host)
|
||||
return groups.items()
|
||||
|
||||
|
||||
def get_certificate_only_hosts(apps, tls_data):
|
||||
"""Hosts the primary server only manages certificates for, because their application is served by workers."""
|
||||
if CLUSTER_NODE:
|
||||
return []
|
||||
served_hosts = {host.lower() for app in apps for host in app.get("hosts", [])}
|
||||
return [
|
||||
host for host, host_tls in tls_data.get("hosts", {}).items()
|
||||
if host_tls.get("certificate_only") and host not in served_hosts
|
||||
]
|
||||
|
||||
|
||||
def build_caddyfile(apps, auth_policies, routes, tls_data=None):
|
||||
tls_data = tls_data or {}
|
||||
policies = auth_policies.get("policies", {}) if auth_policies else {}
|
||||
route_entries = routes.get("entries", {}) if routes else {}
|
||||
lines = []
|
||||
# Caddy refuses an empty Caddyfile, which happens on cluster nodes without applications
|
||||
lines = ["# Generated by process_config.py. Do not edit.", ""]
|
||||
|
||||
def get_policy_type(policy_name):
|
||||
if policy_name and policy_name in policies:
|
||||
@@ -144,6 +180,12 @@ def build_caddyfile(apps, auth_policies, routes, tls_data=None):
|
||||
lines.append("")
|
||||
return
|
||||
|
||||
if certificate_type == "certificate_file":
|
||||
certificate_path = os.path.join(NODE_CERTIFICATES_DIR, dns_credential)
|
||||
lines.append(f" tls {certificate_path}.crt {certificate_path}.key")
|
||||
lines.append("")
|
||||
return
|
||||
|
||||
lines.append(" tls {")
|
||||
if certificate_type == "dns_challenge":
|
||||
dns_credential_data = tls_data["dns_credentials"][dns_credential]
|
||||
@@ -299,6 +341,15 @@ def build_caddyfile(apps, auth_policies, routes, tls_data=None):
|
||||
for (certificate_type, dns_credential, certificate_fallback), group_hosts in group_hosts_by_tls(hosts, tls_data):
|
||||
emit_site_block(app, group_hosts, certificate_type, dns_credential, certificate_fallback)
|
||||
|
||||
certificate_only_hosts = get_certificate_only_hosts(apps, tls_data)
|
||||
for (certificate_type, dns_credential, certificate_fallback), group_hosts in group_hosts_by_tls(certificate_only_hosts, tls_data):
|
||||
# Keeps the certificate managed for cluster workers; the application is not published here
|
||||
lines.append(f"{', '.join(_safe_value(host) for host in group_hosts)} {{")
|
||||
emit_tls(certificate_type, dns_credential, certificate_fallback)
|
||||
lines.append(" respond 404")
|
||||
lines.append("}")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
@@ -306,19 +357,21 @@ def build_tls_state(apps, tls_data):
|
||||
"""Map each host to a fingerprint of the certificate settings used in the generated Caddyfile."""
|
||||
tls_data = tls_data or {}
|
||||
tls_state = {}
|
||||
for app in apps:
|
||||
if not app.get("hosts") or not app.get("upstream"):
|
||||
hosts = [host for app in apps if app.get("upstream") for host in app.get("hosts", [])]
|
||||
hosts += get_certificate_only_hosts(apps, tls_data)
|
||||
for host in hosts:
|
||||
host_tls = get_host_tls(host, tls_data)
|
||||
if host_tls is None:
|
||||
continue
|
||||
for host in app["hosts"]:
|
||||
certificate_type, dns_credential, certificate_fallback = get_host_tls(host, tls_data)
|
||||
credential_hash = ""
|
||||
if dns_credential:
|
||||
credential_data = json.dumps(tls_data["dns_credentials"][dns_credential], sort_keys=True)
|
||||
credential_hash = hashlib.sha256(credential_data.encode()).hexdigest()
|
||||
certificate_generation = tls_data.get("hosts", {}).get(host.lower(), {}).get("certificate_generation", 0)
|
||||
tls_state[host.lower()] = [
|
||||
certificate_type, dns_credential or "", credential_hash, certificate_fallback, certificate_generation
|
||||
]
|
||||
certificate_type, certificate_reference, certificate_fallback = host_tls
|
||||
credential_hash = ""
|
||||
if certificate_type == "dns_challenge":
|
||||
credential_data = json.dumps(tls_data["dns_credentials"][certificate_reference], sort_keys=True)
|
||||
credential_hash = hashlib.sha256(credential_data.encode()).hexdigest()
|
||||
certificate_generation = tls_data.get("hosts", {}).get(host.lower(), {}).get("certificate_generation", 0)
|
||||
tls_state[host.lower()] = [
|
||||
certificate_type, certificate_reference or "", credential_hash, certificate_fallback, certificate_generation
|
||||
]
|
||||
return tls_state
|
||||
|
||||
|
||||
@@ -363,9 +416,22 @@ def sync_tls_state():
|
||||
restart_required = False
|
||||
if applied_state is not None:
|
||||
for host, host_state in pending_state.items():
|
||||
if applied_state.get(host) in (None, host_state):
|
||||
previous_host_state = applied_state.get(host)
|
||||
if previous_host_state == host_state or "/" in host or host in ("", ".", ".."):
|
||||
continue
|
||||
if "/" in host or host in ("", ".", ".."):
|
||||
# A new host normally has nothing cached, unless a fallback certificate was left by an earlier use
|
||||
# of the same hostname (e.g. removed from one application and added to another)
|
||||
if previous_host_state is None and not os.path.isdir(
|
||||
os.path.join(CERTIFICATES_DIR, INTERNAL_ISSUER_DIR_NAME, host)
|
||||
):
|
||||
continue
|
||||
if CLUSTER_NODE:
|
||||
# Certificates received from the primary server replace cached ones only after a restart
|
||||
if host_state[0] == "certificate_file":
|
||||
fallback_certificate_dir = os.path.join(CERTIFICATES_DIR, INTERNAL_ISSUER_DIR_NAME, host)
|
||||
if os.path.isdir(fallback_certificate_dir):
|
||||
shutil.rmtree(fallback_certificate_dir)
|
||||
restart_required = True
|
||||
continue
|
||||
if host_state[0] not in ACME_CERTIFICATE_TYPES:
|
||||
# A cached ACME certificate would keep being served instead of the self-signed one
|
||||
@@ -386,6 +452,90 @@ def sync_tls_state():
|
||||
print("restart")
|
||||
|
||||
|
||||
def reset_fallback_certificates():
|
||||
"""
|
||||
Remove fallback certificates of ACME hosts before Caddy starts, so the ACME certificate is requested again
|
||||
on every start instead of only on the next renewal. If the request fails, the fallback is issued again.
|
||||
"""
|
||||
pending_state = _read_json(TLS_STATE_PENDING_PATH) or {}
|
||||
for host, host_state in pending_state.items():
|
||||
if host_state[0] not in ACME_CERTIFICATE_TYPES or "/" in host or host in ("", ".", ".."):
|
||||
continue
|
||||
fallback_certificate_dir = os.path.join(CERTIFICATES_DIR, INTERNAL_ISSUER_DIR_NAME, host)
|
||||
if os.path.isdir(fallback_certificate_dir):
|
||||
shutil.rmtree(fallback_certificate_dir)
|
||||
print(f"Removed fallback certificate for {host}", file=sys.stderr)
|
||||
|
||||
|
||||
def find_acme_certificate(host):
|
||||
"""Return (certificate_path, key_path) of the newest ACME certificate stored by Caddy for a host."""
|
||||
candidates = []
|
||||
if os.path.isdir(CERTIFICATES_DIR):
|
||||
for issuer_dir_name in os.listdir(CERTIFICATES_DIR):
|
||||
if issuer_dir_name == INTERNAL_ISSUER_DIR_NAME:
|
||||
continue
|
||||
certificate_path = os.path.join(CERTIFICATES_DIR, issuer_dir_name, host, f"{host}.crt")
|
||||
key_path = os.path.join(CERTIFICATES_DIR, issuer_dir_name, host, f"{host}.key")
|
||||
if os.path.isfile(certificate_path) and os.path.isfile(key_path):
|
||||
candidates.append((os.path.getmtime(certificate_path), certificate_path, key_path))
|
||||
if not candidates:
|
||||
return None, None
|
||||
newest = max(candidates)
|
||||
return newest[1], newest[2]
|
||||
|
||||
|
||||
def _copy_if_changed(source_path, destination_path):
|
||||
with open(source_path, "r", encoding="utf-8") as source_file:
|
||||
content = source_file.read()
|
||||
if _read_text(destination_path) == content:
|
||||
return False
|
||||
temp_path = f"{destination_path}.tmp"
|
||||
with open(temp_path, "w", encoding="utf-8") as destination_file:
|
||||
destination_file.write(content)
|
||||
os.chmod(temp_path, 0o600)
|
||||
os.replace(temp_path, destination_path)
|
||||
return True
|
||||
|
||||
|
||||
def _read_text(filepath):
|
||||
try:
|
||||
with open(filepath, "r", encoding="utf-8") as text_file:
|
||||
return text_file.read()
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def export_certificates():
|
||||
"""
|
||||
Copy the ACME certificates of hosts served by cluster workers to CERTIFICATE_EXPORT_DIR, where Django reads
|
||||
them to distribute to the workers. Fallback certificates from the internal CA are never exported.
|
||||
"""
|
||||
if CLUSTER_NODE or not os.path.isdir(CERTIFICATE_EXPORT_DIR):
|
||||
return
|
||||
tls_data = load_json("tls.json") or {}
|
||||
export_hosts = {
|
||||
host for host, host_tls in tls_data.get("hosts", {}).items()
|
||||
if host_tls.get("export_certificate") and "/" not in host and host not in ("", ".", "..")
|
||||
}
|
||||
|
||||
for host in sorted(export_hosts):
|
||||
certificate_path, key_path = find_acme_certificate(host)
|
||||
if not certificate_path:
|
||||
continue
|
||||
host_export_dir = os.path.join(CERTIFICATE_EXPORT_DIR, host)
|
||||
os.makedirs(host_export_dir, exist_ok=True)
|
||||
key_changed = _copy_if_changed(key_path, os.path.join(host_export_dir, "certificate.key"))
|
||||
certificate_changed = _copy_if_changed(certificate_path, os.path.join(host_export_dir, "certificate.crt"))
|
||||
if certificate_changed or key_changed:
|
||||
print(f"Exported certificate for {host}")
|
||||
|
||||
for host in os.listdir(CERTIFICATE_EXPORT_DIR):
|
||||
host_export_dir = os.path.join(CERTIFICATE_EXPORT_DIR, host)
|
||||
if host not in export_hosts and os.path.isdir(host_export_dir):
|
||||
shutil.rmtree(host_export_dir)
|
||||
print(f"Removed exported certificate for {host}")
|
||||
|
||||
|
||||
def main():
|
||||
apps = collect_all_applications()
|
||||
auth_policies = load_json("auth_policies.json")
|
||||
@@ -403,5 +553,9 @@ def main():
|
||||
if __name__ == "__main__":
|
||||
if "--sync-tls-state" in sys.argv:
|
||||
sync_tls_state()
|
||||
elif "--export-certificates" in sys.argv:
|
||||
export_certificates()
|
||||
elif "--reset-fallback-certificates" in sys.argv:
|
||||
reset_fallback_certificates()
|
||||
else:
|
||||
main()
|
||||
|
||||
@@ -12,6 +12,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
inetutils-traceroute \
|
||||
procps \
|
||||
curl \
|
||||
openssl \
|
||||
nano \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
|
||||
@@ -1,350 +1,432 @@
|
||||
import glob
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import ssl
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from typing import Dict, Any
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import requests
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Constants
|
||||
MASTER_SERVER_ADDRESS = os.environ.get('MASTER_SERVER_ADDRESS')
|
||||
TOKEN = os.environ.get('TOKEN')
|
||||
MASTER_SERVER_ADDRESS = os.environ.get('MASTER_SERVER_ADDRESS', '')
|
||||
TOKEN = os.environ.get('TOKEN', '')
|
||||
WORKER_VERSION = 12
|
||||
REQUEST_TIMEOUT = 30
|
||||
MANIFEST_INTERVAL = 60
|
||||
DEFAULT_STATS_INTERVAL = 60
|
||||
|
||||
WIREGUARD_DIR = '/etc/wireguard'
|
||||
FIREWALL_SCRIPT = os.path.join(WIREGUARD_DIR, 'wg-firewall.sh')
|
||||
DNS_DIR = '/etc/dnsmasq'
|
||||
WORKER_VERSION = 11
|
||||
REQUEST_TIMEOUT = 20
|
||||
|
||||
class ClusterWorker:
|
||||
def __init__(self):
|
||||
self.config_version = 0
|
||||
self.dns_version = self.get_local_dns_version()
|
||||
self.should_run = True
|
||||
self.session = requests.Session()
|
||||
|
||||
if not MASTER_SERVER_ADDRESS or not TOKEN:
|
||||
logger.error("MASTER_SERVER_ADDRESS or TOKEN not set")
|
||||
self.should_run = False
|
||||
|
||||
self.base_url = f"https://{MASTER_SERVER_ADDRESS}/api/cluster"
|
||||
# Validate URL scheme
|
||||
if not MASTER_SERVER_ADDRESS.startswith(('http://', 'https://')):
|
||||
self.base_url = f"https://{MASTER_SERVER_ADDRESS}/api/cluster"
|
||||
else:
|
||||
self.base_url = f"{MASTER_SERVER_ADDRESS}/api/cluster"
|
||||
DNS_MAIN_CONFIG = 'wireguard_webadmin_dns.conf'
|
||||
# Only mounted when the node runs the App Gateway (cluster_node-gateway.yml)
|
||||
GATEWAY_DIR = '/caddy_json_export'
|
||||
CERTIFICATES_DIR = '/caddy_certificates'
|
||||
GATEWAY_FILES = ('applications.json', 'auth_policies.json', 'routes.json', 'tls.json')
|
||||
CERTIFICATE_FILE_PATTERN = re.compile(r'^[a-f0-9]{64}\.(crt|key)$')
|
||||
CADDY_ADMIN_URL = 'http://127.0.0.1:2019/config/'
|
||||
AUTH_GATEWAY_URL = 'http://cluster-node-auth-gateway:9091/'
|
||||
INTERNAL_CA_NAME = 'Caddy Local Authority'
|
||||
FIREWALL_CHAINS = (
|
||||
('nat', 'WGWADM_POSTROUTING'),
|
||||
('nat', 'WGWADM_PREROUTING'),
|
||||
('filter', 'WGWADM_FORWARD'),
|
||||
('filter', 'WGWADM_ROUTE_POLICY'),
|
||||
)
|
||||
|
||||
|
||||
def func_process_wireguard_status(self) -> Dict[str, Any]:
|
||||
command = "wg show all dump"
|
||||
|
||||
process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
stdout, stderr = process.communicate()
|
||||
|
||||
if process.returncode != 0:
|
||||
return {"message": stderr, "status": "error"}
|
||||
|
||||
data: Dict[str, Dict[str, Dict[str, Any]]] = {}
|
||||
|
||||
# wg dump format is tab-separated.
|
||||
# There are two kinds of lines:
|
||||
# - Interface line: interface \t private_key \t public_key \t listen_port \t fwmark
|
||||
# - Peer line: interface \t peer_public_key \t preshared_key \t endpoint \t allowed_ips \t latest_handshake \t transfer_rx \t transfer_tx \t persistent_keepalive
|
||||
for line in stdout.strip().split("\n"):
|
||||
parts = line.split("\t")
|
||||
if len(parts) < 5:
|
||||
continue
|
||||
|
||||
interface = parts[0]
|
||||
|
||||
# Peer lines are expected to have at least 9 fields
|
||||
if len(parts) < 9:
|
||||
# interface line, ignore
|
||||
continue
|
||||
|
||||
peer_public_key = parts[1]
|
||||
endpoint = parts[3]
|
||||
allowed_ips_raw = parts[4]
|
||||
latest_handshake = parts[5]
|
||||
transfer_rx = parts[6]
|
||||
transfer_tx = parts[7]
|
||||
|
||||
if interface not in data:
|
||||
data[interface] = {}
|
||||
|
||||
data[interface][peer_public_key] = {
|
||||
"allowed-ips": [ip for ip in allowed_ips_raw.split(",") if ip] if allowed_ips_raw else [],
|
||||
"latest-handshakes": latest_handshake or "",
|
||||
"transfer": {"tx": int(transfer_tx or 0), "rx": int(transfer_rx or 0)},
|
||||
"endpoints": endpoint or "",
|
||||
}
|
||||
|
||||
return data
|
||||
class SectionError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def cleanup_wireguard(self):
|
||||
logger.info("Cleaning up WireGuard configuration...")
|
||||
# Stop all wireguard interfaces
|
||||
try:
|
||||
files = glob.glob(f"{WIREGUARD_DIR}/*.conf")
|
||||
for f in files:
|
||||
interface = os.path.basename(f).replace('.conf', '')
|
||||
subprocess.run(['wg-quick', 'down', interface], capture_output=True)
|
||||
except Exception as e:
|
||||
logger.error(f"Error stopping wireguard interfaces: {e}")
|
||||
def run_command(command, check=True):
|
||||
result = subprocess.run(command, capture_output=True, text=True)
|
||||
if check and result.returncode != 0:
|
||||
raise SectionError(f"{' '.join(command)}: {(result.stderr or result.stdout).strip()}")
|
||||
return result
|
||||
|
||||
# Remove files
|
||||
try:
|
||||
for f in glob.glob(f"{WIREGUARD_DIR}/*"):
|
||||
os.remove(f)
|
||||
except Exception as e:
|
||||
logger.error(f"Error cleaning directory: {e}")
|
||||
|
||||
try:
|
||||
subprocess.run(['iptables', '-t', 'nat', '-F', 'WGWADM_POSTROUTING'], capture_output=True)
|
||||
subprocess.run(['iptables', '-t', 'nat', '-F', 'WGWADM_PREROUTING'], capture_output=True)
|
||||
subprocess.run(['iptables', '-t', 'filter', '-F', 'WGWADM_FORWARD'], capture_output=True)
|
||||
except Exception as e:
|
||||
logger.error(f"Error flushing firewall: {e}")
|
||||
def interface_is_up(interface):
|
||||
return run_command(['wg', 'show', interface], check=False).returncode == 0
|
||||
|
||||
def get_local_dns_version(self):
|
||||
try:
|
||||
version_file = os.path.join(DNS_DIR, 'config_version.conf')
|
||||
if os.path.exists(version_file):
|
||||
with open(version_file, 'r') as f:
|
||||
for line in f:
|
||||
if line.startswith('DNS_VERSION='):
|
||||
return int(line.strip().split('=')[1])
|
||||
except Exception as e:
|
||||
logger.error(f"Error reading DNS version: {e}")
|
||||
return 0
|
||||
|
||||
def get_status(self):
|
||||
params = {
|
||||
'token': TOKEN,
|
||||
'worker_config_version': self.config_version,
|
||||
'worker_dns_version': self.dns_version,
|
||||
'worker_version': WORKER_VERSION
|
||||
def interface_section(config):
|
||||
"""Return the [Interface] part of a WireGuard config, used to decide between syncconf and restart."""
|
||||
return config.split('[Peer]', 1)[0].strip()
|
||||
|
||||
|
||||
def read_file(path):
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as config_file:
|
||||
return config_file.read()
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
|
||||
|
||||
def write_file(path, content, mode=0o600, in_place=False):
|
||||
if in_place:
|
||||
with open(path, 'w', encoding='utf-8') as config_file:
|
||||
config_file.write(content)
|
||||
os.chmod(path, mode)
|
||||
return
|
||||
temp_path = f'{path}.tmp'
|
||||
with open(temp_path, 'w', encoding='utf-8') as config_file:
|
||||
config_file.write(content)
|
||||
os.chmod(temp_path, mode)
|
||||
os.replace(temp_path, path)
|
||||
|
||||
|
||||
def wireguard_status():
|
||||
result = run_command(['wg', 'show', 'all', 'dump'], check=False)
|
||||
if result.returncode != 0:
|
||||
return {}
|
||||
|
||||
data = {}
|
||||
# Peer lines: interface, public_key, preshared_key, endpoint, allowed_ips, latest_handshake, rx, tx, keepalive
|
||||
for line in result.stdout.strip().splitlines():
|
||||
parts = line.split('\t')
|
||||
if len(parts) < 9:
|
||||
continue
|
||||
interface, peer_public_key, endpoint, allowed_ips = parts[0], parts[1], parts[3], parts[4]
|
||||
data.setdefault(interface, {})[peer_public_key] = {
|
||||
'allowed-ips': [ip for ip in allowed_ips.split(',') if ip] if allowed_ips else [],
|
||||
'latest-handshakes': parts[5] or '',
|
||||
'transfer': {'tx': int(parts[7] or 0), 'rx': int(parts[6] or 0)},
|
||||
'endpoints': endpoint or '',
|
||||
}
|
||||
logger.info(f"Requesting status from Master... (Config Version: {self.config_version}, DNS Version: {self.dns_version}, Worker Version: {WORKER_VERSION})")
|
||||
try:
|
||||
response = self.session.get(f"{self.base_url}/status/", params=params, timeout=REQUEST_TIMEOUT)
|
||||
logger.info(f"Status response received. HTTP Code: {response.status_code}")
|
||||
return response
|
||||
except requests.RequestException as e:
|
||||
logger.error(f"Connection error while getting status: {e}")
|
||||
return None
|
||||
return data
|
||||
|
||||
def download_configs(self):
|
||||
params = {
|
||||
'token': TOKEN,
|
||||
'worker_config_version': self.config_version,
|
||||
'worker_dns_version': self.dns_version,
|
||||
'worker_version': WORKER_VERSION
|
||||
|
||||
def gateway_enabled():
|
||||
return os.path.isdir(GATEWAY_DIR) and os.path.isdir(CERTIFICATES_DIR)
|
||||
|
||||
|
||||
def http_service_is_running(url):
|
||||
try:
|
||||
requests.get(url, timeout=3)
|
||||
return True
|
||||
except requests.RequestException:
|
||||
return False
|
||||
|
||||
|
||||
def _parse_not_after(value):
|
||||
return datetime.strptime(value, '%b %d %H:%M:%S %Y %Z').replace(tzinfo=timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _fetch_certificate(hostname, verify):
|
||||
if verify:
|
||||
context = ssl.create_default_context()
|
||||
else:
|
||||
context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
|
||||
context.check_hostname = False
|
||||
context.verify_mode = ssl.CERT_NONE
|
||||
with socket.create_connection(('127.0.0.1', 443), timeout=3) as raw_socket:
|
||||
with context.wrap_socket(raw_socket, server_hostname=hostname) as tls_socket:
|
||||
return tls_socket.getpeercert() if verify else tls_socket.getpeercert(binary_form=True)
|
||||
|
||||
|
||||
def inspect_certificate(hostname):
|
||||
"""Same checks as app_gateway/certificate_status.py on the primary server, against the local Caddy."""
|
||||
try:
|
||||
certificate = _fetch_certificate(hostname, verify=True)
|
||||
issuer = dict(field[0] for field in certificate.get('issuer', ()))
|
||||
return {
|
||||
'status': 'trusted',
|
||||
'issuer': issuer.get('organizationName') or issuer.get('commonName') or '',
|
||||
'not_after': _parse_not_after(certificate['notAfter']),
|
||||
}
|
||||
try:
|
||||
response = self.session.get(f"{self.base_url}/worker/get_config_files/", params=params, timeout=REQUEST_TIMEOUT)
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
if data.get('status') == 'success':
|
||||
return data
|
||||
return None
|
||||
except requests.RequestException as e:
|
||||
logger.error(f"Error downloading configs: {e}")
|
||||
return None
|
||||
except ssl.SSLCertVerificationError:
|
||||
pass
|
||||
except (OSError, ValueError):
|
||||
return {'status': 'unavailable', 'issuer': '', 'not_after': None}
|
||||
|
||||
def download_dns_config(self):
|
||||
params = {
|
||||
'token': TOKEN,
|
||||
'worker_config_version': self.config_version,
|
||||
'worker_dns_version': self.dns_version,
|
||||
'worker_version': WORKER_VERSION
|
||||
}
|
||||
try:
|
||||
logger.info(f"Downloading DNS config... (Current Version: {self.dns_version})")
|
||||
response = self.session.get(f"{self.base_url}/worker/get_dnsmasq_config/", params=params, timeout=REQUEST_TIMEOUT)
|
||||
if response.status_code == 200:
|
||||
tar_path = 'dnsmasq_config.tar.gz'
|
||||
with open(tar_path, 'wb') as f:
|
||||
f.write(response.content)
|
||||
|
||||
logger.info("Extracting DNS config...")
|
||||
if not os.path.exists(DNS_DIR):
|
||||
os.makedirs(DNS_DIR, exist_ok=True)
|
||||
|
||||
subprocess.run(['tar', 'xvfz', tar_path, '-C', DNS_DIR], check=True, capture_output=True)
|
||||
if os.path.exists(tar_path):
|
||||
os.remove(tar_path)
|
||||
|
||||
# Update version
|
||||
self.dns_version = self.get_local_dns_version()
|
||||
logger.info(f"DNS config updated (New Version: {self.dns_version})")
|
||||
return True
|
||||
else:
|
||||
logger.error(f"Failed to download DNS config. Status: {response.status_code}")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating DNS config: {e}")
|
||||
return False
|
||||
try:
|
||||
certificate_der = _fetch_certificate(hostname, verify=False)
|
||||
result = subprocess.run(
|
||||
['openssl', 'x509', '-inform', 'DER', '-noout', '-issuer', '-enddate', '-nameopt', 'RFC2253'],
|
||||
input=certificate_der, capture_output=True, timeout=5,
|
||||
)
|
||||
except (OSError, ValueError, subprocess.SubprocessError):
|
||||
return {'status': 'unavailable', 'issuer': '', 'not_after': None}
|
||||
issuer = ''
|
||||
not_after = None
|
||||
for line in result.stdout.decode(errors='replace').splitlines():
|
||||
if line.startswith('issuer='):
|
||||
parts = dict(part.split('=', 1) for part in line[len('issuer='):].split(',') if '=' in part)
|
||||
issuer = INTERNAL_CA_NAME if INTERNAL_CA_NAME in line else parts.get('O') or parts.get('CN') or ''
|
||||
elif line.startswith('notAfter='):
|
||||
not_after = _parse_not_after(line[len('notAfter='):])
|
||||
return {'status': 'untrusted', 'issuer': issuer, 'not_after': not_after}
|
||||
|
||||
def send_ping(self):
|
||||
params = {
|
||||
'token': TOKEN,
|
||||
'worker_config_version': self.config_version,
|
||||
'worker_dns_version': self.dns_version,
|
||||
'worker_version': WORKER_VERSION
|
||||
}
|
||||
try:
|
||||
logger.info("Sending ping to Master...")
|
||||
self.session.get(f"{self.base_url}/worker/ping/", params=params, timeout=REQUEST_TIMEOUT)
|
||||
except Exception as e:
|
||||
logger.error(f"Error sending ping: {e}")
|
||||
|
||||
def apply_configs(self, data):
|
||||
logger.info("Applying new configurations...")
|
||||
files = data.get('files', {})
|
||||
cluster_settings = data.get('cluster_settings', {})
|
||||
new_config_version = cluster_settings.get('config_version', 0)
|
||||
|
||||
# 1. Stop existing interfaces
|
||||
self.cleanup_wireguard()
|
||||
|
||||
# 2. Write new files
|
||||
for filename, content in files.items():
|
||||
if filename == 'wg-firewall.sh' and isinstance(content, str):
|
||||
content = content.replace('wireguard-webadmin-dns', 'cluster-node-dns')
|
||||
filepath = os.path.join(WIREGUARD_DIR, filename)
|
||||
with open(filepath, 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
if filename == 'wg-firewall.sh':
|
||||
os.chmod(filepath, 0o755)
|
||||
|
||||
# Start interfaces
|
||||
conf_files = glob.glob(f"{WIREGUARD_DIR}/*.conf")
|
||||
for conf in conf_files:
|
||||
interface = os.path.basename(conf).replace('.conf', '')
|
||||
logger.info(f"Starting WireGuard interface: {interface}")
|
||||
def dns_is_running():
|
||||
# dnsmasq always listens on loopback when it is restricted to specific interfaces. The DNS container pauses
|
||||
# for a few seconds while reloading, so a missing answer is retried before reporting it as stopped.
|
||||
query = b'\x12\x34\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x09localhost\x00\x00\x01\x00\x01'
|
||||
for attempt in range(4):
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as dns_socket:
|
||||
dns_socket.settimeout(2)
|
||||
try:
|
||||
subprocess.run(['wg-quick', 'up', interface], check=True, capture_output=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.error(f"Failed to start {interface}: {e.stderr.decode()}")
|
||||
dns_socket.sendto(query, ('127.0.0.1', 53))
|
||||
dns_socket.recvfrom(512)
|
||||
return True
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(1)
|
||||
return False
|
||||
|
||||
# 4. Update config version
|
||||
self.config_version = new_config_version
|
||||
logger.info(f"Configuration updated to version {self.config_version}")
|
||||
|
||||
class ClusterNodeAgent:
|
||||
def __init__(self):
|
||||
self.base_url = f"{MASTER_SERVER_ADDRESS.rstrip('/')}/api/cluster/v2"
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update({
|
||||
'Authorization': f'Bearer {TOKEN}',
|
||||
'X-Worker-Version': str(WORKER_VERSION),
|
||||
})
|
||||
# Nothing is persisted: after a restart interfaces and firewall rules are gone, so every section is applied again
|
||||
self.applied_hashes = {}
|
||||
self.stats_interval = DEFAULT_STATS_INTERVAL
|
||||
self.wireguard_active = False
|
||||
|
||||
def request(self, method, path, **kwargs):
|
||||
try:
|
||||
return self.session.request(method, f'{self.base_url}/{path}', timeout=REQUEST_TIMEOUT, **kwargs)
|
||||
except requests.RequestException as error:
|
||||
logger.error('Request to %s failed: %s', path, error)
|
||||
return None
|
||||
|
||||
# Section handlers
|
||||
|
||||
def apply_firewall(self, section):
|
||||
content = section['files'].get('wg-firewall.sh')
|
||||
if content is None:
|
||||
raise SectionError('wg-firewall.sh missing from section')
|
||||
write_file(FIREWALL_SCRIPT, content, mode=0o700)
|
||||
run_command(['bash', FIREWALL_SCRIPT])
|
||||
|
||||
def apply_wireguard(self, section):
|
||||
desired_files = {
|
||||
filename: content for filename, content in section['files'].items()
|
||||
if filename.startswith('wg') and filename.endswith('.conf') and '/' not in filename
|
||||
}
|
||||
errors = []
|
||||
|
||||
for config_path in glob.glob(os.path.join(WIREGUARD_DIR, 'wg[0-9]*.conf')):
|
||||
filename = os.path.basename(config_path)
|
||||
if filename in desired_files:
|
||||
continue
|
||||
interface = filename[:-len('.conf')]
|
||||
if interface_is_up(interface):
|
||||
run_command(['wg-quick', 'down', interface], check=False)
|
||||
os.remove(config_path)
|
||||
logger.info('Removed WireGuard interface %s', interface)
|
||||
|
||||
for filename, content in sorted(desired_files.items()):
|
||||
interface = filename[:-len('.conf')]
|
||||
config_path = os.path.join(WIREGUARD_DIR, filename)
|
||||
previous_content = read_file(config_path)
|
||||
write_file(config_path, content)
|
||||
try:
|
||||
if not interface_is_up(interface):
|
||||
run_command(['wg-quick', 'up', interface])
|
||||
logger.info('Started WireGuard interface %s', interface)
|
||||
elif previous_content is None or interface_section(previous_content) != interface_section(content):
|
||||
run_command(['wg-quick', 'down', interface], check=False)
|
||||
run_command(['wg-quick', 'up', interface])
|
||||
logger.info('Restarted WireGuard interface %s', interface)
|
||||
elif previous_content != content:
|
||||
self.sync_peers(interface, config_path)
|
||||
logger.info('Synchronized peers of WireGuard interface %s', interface)
|
||||
except SectionError as error:
|
||||
errors.append(str(error))
|
||||
|
||||
self.wireguard_active = True
|
||||
if errors:
|
||||
raise SectionError('; '.join(errors))
|
||||
|
||||
@staticmethod
|
||||
def sync_peers(interface, config_path):
|
||||
stripped_config = run_command(['wg-quick', 'strip', config_path]).stdout
|
||||
with tempfile.NamedTemporaryFile('w', suffix='.conf', delete=False) as stripped_file:
|
||||
stripped_file.write(stripped_config)
|
||||
try:
|
||||
run_command(['wg', 'syncconf', interface, stripped_file.name])
|
||||
finally:
|
||||
os.remove(stripped_file.name)
|
||||
|
||||
def apply_dns(self, section):
|
||||
files = section['files']
|
||||
if DNS_MAIN_CONFIG not in files:
|
||||
raise SectionError(f'{DNS_MAIN_CONFIG} missing from section')
|
||||
os.makedirs(DNS_DIR, exist_ok=True)
|
||||
for filename, content in files.items():
|
||||
if filename != DNS_MAIN_CONFIG and '/' not in filename:
|
||||
write_file(os.path.join(DNS_DIR, filename), content, mode=0o644)
|
||||
for config_path in glob.glob(os.path.join(DNS_DIR, '*.conf')):
|
||||
if os.path.basename(config_path) not in files:
|
||||
os.remove(config_path)
|
||||
# Written last and in place: the DNS container watches this file with inotify to reload
|
||||
write_file(os.path.join(DNS_DIR, DNS_MAIN_CONFIG), files[DNS_MAIN_CONFIG], mode=0o644, in_place=True)
|
||||
|
||||
def apply_settings(self, section):
|
||||
self.stats_interval = int(section['values'].get('stats_sync_interval', DEFAULT_STATS_INTERVAL))
|
||||
|
||||
def apply_certificates(self, section):
|
||||
if not gateway_enabled():
|
||||
return
|
||||
files = {
|
||||
filename: content for filename, content in section['files'].items()
|
||||
if CERTIFICATE_FILE_PATTERN.match(filename)
|
||||
}
|
||||
for filename, content in files.items():
|
||||
certificate_path = os.path.join(CERTIFICATES_DIR, filename)
|
||||
if read_file(certificate_path) != content:
|
||||
write_file(certificate_path, content, mode=0o600)
|
||||
for certificate_path in glob.glob(os.path.join(CERTIFICATES_DIR, '*')):
|
||||
if os.path.basename(certificate_path) not in files:
|
||||
os.remove(certificate_path)
|
||||
|
||||
def apply_gateway(self, section):
|
||||
if not gateway_enabled():
|
||||
return
|
||||
for filename in GATEWAY_FILES:
|
||||
if filename not in section['files']:
|
||||
raise SectionError(f'{filename} missing from section')
|
||||
for filename in GATEWAY_FILES:
|
||||
write_file(os.path.join(GATEWAY_DIR, filename), section['files'][filename], mode=0o644)
|
||||
|
||||
SECTION_HANDLERS = {
|
||||
'firewall': apply_firewall,
|
||||
'wireguard': apply_wireguard,
|
||||
'dns': apply_dns,
|
||||
'settings': apply_settings,
|
||||
'certificates': apply_certificates,
|
||||
'gateway': apply_gateway,
|
||||
}
|
||||
|
||||
# Main loop
|
||||
|
||||
def shutdown_wireguard(self):
|
||||
if not self.wireguard_active:
|
||||
return
|
||||
logger.warning('Shutting down WireGuard interfaces and firewall rules')
|
||||
for config_path in glob.glob(os.path.join(WIREGUARD_DIR, 'wg[0-9]*.conf')):
|
||||
run_command(['wg-quick', 'down', os.path.basename(config_path)[:-len('.conf')]], check=False)
|
||||
for table, chain in FIREWALL_CHAINS:
|
||||
run_command(['iptables', '-t', table, '-F', chain], check=False)
|
||||
self.applied_hashes.pop('wireguard', None)
|
||||
self.applied_hashes.pop('firewall', None)
|
||||
self.wireguard_active = False
|
||||
|
||||
def sync_configuration(self):
|
||||
response = self.request('GET', 'manifest/')
|
||||
if response is None:
|
||||
return True
|
||||
if response.status_code == 403:
|
||||
logger.error('Token rejected by the primary server. Stopping.')
|
||||
self.shutdown_wireguard()
|
||||
return False
|
||||
if response.status_code == 400:
|
||||
logger.warning('Primary server refused this worker: %s', response.json().get('message', ''))
|
||||
self.shutdown_wireguard()
|
||||
return True
|
||||
if response.status_code != 200:
|
||||
logger.error('Unexpected manifest response: HTTP %s', response.status_code)
|
||||
return True
|
||||
|
||||
manifest = response.json().get('sections', {})
|
||||
reports = {}
|
||||
for section_name, section_hash in manifest.items():
|
||||
if section_name not in self.SECTION_HANDLERS or self.applied_hashes.get(section_name) == section_hash:
|
||||
continue
|
||||
section_response = self.request('GET', f'section/{section_name}/')
|
||||
if section_response is None or section_response.status_code != 200:
|
||||
continue
|
||||
section = section_response.json()
|
||||
try:
|
||||
self.SECTION_HANDLERS[section_name](self, section)
|
||||
self.applied_hashes[section_name] = section['hash']
|
||||
reports[section_name] = {'hash': section['hash'], 'status': 'ok'}
|
||||
logger.info('Section %s applied', section_name)
|
||||
except (SectionError, OSError, ValueError) as error:
|
||||
# Forget the last applied content: the next version must be applied and reported even if it
|
||||
# is identical to the one that was applied before the failure
|
||||
self.applied_hashes.pop(section_name, None)
|
||||
reports[section_name] = {'hash': section['hash'], 'status': 'error', 'message': str(error)}
|
||||
logger.error('Section %s failed: %s', section_name, error)
|
||||
|
||||
if reports:
|
||||
# Services are reported with the periodic stats: right after a DNS change dnsmasq is still reloading
|
||||
self.request('POST', 'report/', json={'sections': reports})
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def services_status():
|
||||
services = {'dnsmasq': 'running' if dns_is_running() else 'stopped'}
|
||||
if gateway_enabled():
|
||||
services['caddy'] = 'running' if http_service_is_running(CADDY_ADMIN_URL) else 'stopped'
|
||||
services['auth_gateway'] = 'running' if http_service_is_running(AUTH_GATEWAY_URL) else 'stopped'
|
||||
return services
|
||||
|
||||
@staticmethod
|
||||
def certificates_status():
|
||||
try:
|
||||
with open(os.path.join(GATEWAY_DIR, 'tls.json'), 'r', encoding='utf-8') as tls_file:
|
||||
hosts = json.load(tls_file).get('hosts', {})
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
return {hostname: inspect_certificate(hostname) for hostname in sorted(hosts)}
|
||||
|
||||
def send_stats(self):
|
||||
try:
|
||||
stats = self.func_process_wireguard_status()
|
||||
params = {
|
||||
'token': TOKEN,
|
||||
'worker_config_version': self.config_version,
|
||||
'worker_dns_version': self.dns_version,
|
||||
'worker_version': WORKER_VERSION
|
||||
}
|
||||
logger.info("Sending WireGuard stats to Master...")
|
||||
response = self.session.post(f"{self.base_url}/worker/submit_wireguard_stats/", json=stats, params=params, timeout=REQUEST_TIMEOUT)
|
||||
|
||||
if response.status_code == 200:
|
||||
logger.info("Stats sent successfully.")
|
||||
return True
|
||||
else:
|
||||
logger.error(f"Failed to send stats. Status: {response.status_code}")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Error sending stats: {e}")
|
||||
return False
|
||||
report = {
|
||||
'wireguard_status': wireguard_status(),
|
||||
'services': self.services_status(),
|
||||
}
|
||||
if gateway_enabled():
|
||||
report['certificates'] = self.certificates_status()
|
||||
self.request('POST', 'report/', json=report)
|
||||
|
||||
@staticmethod
|
||||
def publish_network_namespace():
|
||||
# Containers sharing this network namespace (DNS, Caddy) restart themselves when it changes
|
||||
network_namespace = os.readlink('/proc/self/ns/net')
|
||||
for directory in (DNS_DIR, GATEWAY_DIR):
|
||||
if os.path.isdir(directory):
|
||||
write_file(os.path.join(directory, '.netns_owner'), network_namespace + '\n', mode=0o644)
|
||||
|
||||
def run(self):
|
||||
if not self.should_run:
|
||||
return
|
||||
|
||||
logger.info("Cluster Worker starting...")
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_wireguard()
|
||||
|
||||
last_config_check = 0
|
||||
last_stats_send = 0
|
||||
stats_sync_interval = 60 # Default initial value
|
||||
logger.info('Cluster node agent %s starting', WORKER_VERSION)
|
||||
self.publish_network_namespace()
|
||||
self.wireguard_active = True
|
||||
self.shutdown_wireguard()
|
||||
|
||||
last_manifest = 0
|
||||
last_stats = 0
|
||||
while True:
|
||||
current_time = time.time()
|
||||
|
||||
# Check Config (Fixed 60s interval)
|
||||
if current_time - last_config_check >= 60:
|
||||
try:
|
||||
response = self.get_status()
|
||||
last_config_check = time.time()
|
||||
|
||||
if response is not None:
|
||||
if response.status_code == 403:
|
||||
logger.error("Received 403 Forbidden (Token invalid/deleted). Deactivating WireGuard and stopping requests permanently.")
|
||||
self.cleanup_wireguard()
|
||||
self.config_version = 0
|
||||
self.should_run = False
|
||||
break
|
||||
|
||||
if response.status_code == 400:
|
||||
logger.warning("Received 400 Bad Request (Worker suspended or error). Deactivating WireGuard and Firewall, but will keep retrying...")
|
||||
self.cleanup_wireguard()
|
||||
self.config_version = 0
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
|
||||
# Update stats interval from master settings
|
||||
cluster_settings = data.get('cluster_settings', {})
|
||||
stats_sync_interval = cluster_settings.get('stats_sync_interval', 60)
|
||||
|
||||
remote_config_version = cluster_settings.get('config_version', 0)
|
||||
|
||||
# Check WireGuard Config
|
||||
updated = False
|
||||
if int(remote_config_version) != self.config_version:
|
||||
logger.info(f"Config version mismatch (Local: {self.config_version}, Remote: {remote_config_version}). Updating...")
|
||||
config_data = self.download_configs()
|
||||
if config_data:
|
||||
self.apply_configs(config_data)
|
||||
self.send_ping()
|
||||
updated = True
|
||||
else:
|
||||
logger.error("Failed to download config files.")
|
||||
|
||||
# Check DNS Config
|
||||
remote_dns_version = int(cluster_settings.get('dns_version', 0))
|
||||
if not updated and remote_dns_version != self.dns_version:
|
||||
logger.info(f"DNS version mismatch (Local: {self.dns_version}, Remote: {remote_dns_version}). Updating...")
|
||||
if self.download_dns_config():
|
||||
self.send_ping()
|
||||
updated = True
|
||||
|
||||
if not updated and int(remote_config_version) == self.config_version and remote_dns_version == self.dns_version:
|
||||
logger.info(f"No changes detected. Configuration is up to date (WG: {self.config_version}, DNS: {self.dns_version}).")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error in config check loop: {e}")
|
||||
|
||||
# Check Stats Sync
|
||||
if current_time - last_stats_send >= stats_sync_interval:
|
||||
now = time.monotonic()
|
||||
if now - last_manifest >= MANIFEST_INTERVAL or not last_manifest:
|
||||
last_manifest = now
|
||||
if not self.sync_configuration():
|
||||
break
|
||||
if now - last_stats >= self.stats_interval:
|
||||
last_stats = now
|
||||
self.send_stats()
|
||||
last_stats_send = time.time()
|
||||
|
||||
# Sleep briefly to be responsive but not busy loop
|
||||
time.sleep(1)
|
||||
|
||||
# Final loop state if 403 was received
|
||||
while not self.should_run:
|
||||
logger.info("Worker disabled due to 403 Forbidden. WireGuard is off.")
|
||||
time.sleep(60)
|
||||
while True:
|
||||
time.sleep(3600)
|
||||
|
||||
if __name__ == "__main__":
|
||||
worker = ClusterWorker()
|
||||
worker.run()
|
||||
|
||||
if __name__ == '__main__':
|
||||
if not MASTER_SERVER_ADDRESS or not TOKEN:
|
||||
logger.error('MASTER_SERVER_ADDRESS and TOKEN must be set')
|
||||
else:
|
||||
ClusterNodeAgent().run()
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
if [[ "$COMPOSE_VERSION" != "c1b" ]]; then
|
||||
if [[ "$COMPOSE_VERSION" != "c2a" ]]; then
|
||||
echo "ERROR: Please upgrade your docker compose file. Exiting."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -34,7 +34,24 @@ handle_sigint() {
|
||||
exit 0
|
||||
}
|
||||
|
||||
# Written by cluster nodes, where this container shares the network namespace of the node. When the node
|
||||
# container is restarted it gets a new namespace, so this container must restart to join it again.
|
||||
NETNS_OWNER_FILE="/etc/dnsmasq/.netns_owner"
|
||||
|
||||
watch_network_namespace() {
|
||||
while sleep 30; do
|
||||
if [ -f "$NETNS_OWNER_FILE" ] && [ "$(cat "$NETNS_OWNER_FILE")" != "$(readlink /proc/self/ns/net)" ]; then
|
||||
echo "Network namespace of the cluster node changed, restarting..."
|
||||
kill -TERM 1
|
||||
pkill inotifywait
|
||||
return
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
trap handle_sigint SIGINT
|
||||
trap 'pkill dnsmasq; exit 1' SIGTERM
|
||||
|
||||
create_default_config
|
||||
watch_network_namespace &
|
||||
start_dnsmasq
|
||||
|
||||
Reference in New Issue
Block a user