Rework cluster into v2: section-based worker sync and App Gateway on workers

This commit is contained in:
Eduardo Silva
2026-11-20 11:56:30 -03:00
parent dee270695d
commit 7468cdf623
41 changed files with 2766 additions and 1014 deletions
@@ -0,0 +1,40 @@
{
"hosts": {
"dashboard.example.test": {
"certificate_type": "http_challenge",
"certificate_fallback": true,
"certificate_generation": 0
},
"hypervisor.example.test": {
"certificate_type": "http_challenge",
"certificate_fallback": true,
"certificate_generation": 0
},
"media.example.test": {
"certificate_type": "http_challenge",
"certificate_fallback": true,
"certificate_generation": 0
},
"media2.example.test": {
"certificate_type": "http_challenge",
"certificate_fallback": true,
"certificate_generation": 0
},
"sso.example.test": {
"certificate_type": "http_challenge",
"certificate_fallback": true,
"certificate_generation": 0
},
"webadmin.example.test": {
"certificate_type": "http_challenge",
"certificate_fallback": true,
"certificate_generation": 0
},
"webadmin2.example.test": {
"certificate_type": "http_challenge",
"certificate_fallback": true,
"certificate_generation": 0
}
},
"dns_credentials": {}
}
+50 -10
View File
@@ -7,16 +7,20 @@ PYTHON="python3"
JSON_DIR="${JSON_DIR:-/caddy_json_export}"
CADDYFILE_PATH="${CADDYFILE_PATH:-/etc/caddy/Caddyfile}"
CADDY_PID=""
CLUSTER_NODE="${CLUSTER_NODE:-false}"
echo "==> Generating wireguard_webadmin.json..."
"$PYTHON" "${SCRIPT_DIR}/export_wireguard_webadmin_config.py"
# The WireGuard WebAdmin interface is only published by the primary server
if [ "$CLUSTER_NODE" != "true" ]; then
echo "==> Generating wireguard_webadmin.json..."
"$PYTHON" "${SCRIPT_DIR}/export_wireguard_webadmin_config.py"
fi
generate_caddyfile() {
"$PYTHON" "${SCRIPT_DIR}/process_config.py"
"$PYTHON" "${SCRIPT_DIR}/process_config.py" >/dev/null
# Validation output is not printed because provider errors may include credentials
if ! caddy validate --config "$CADDYFILE_PATH" --adapter caddyfile >/dev/null 2>&1; then
echo "Warning: generated Caddyfile is invalid, regenerating with DNS challenge disabled. Check the DNS credentials."
DISABLE_DNS_CHALLENGE=1 "$PYTHON" "${SCRIPT_DIR}/process_config.py"
DISABLE_DNS_CHALLENGE=1 "$PYTHON" "${SCRIPT_DIR}/process_config.py" >/dev/null
fi
}
@@ -42,27 +46,63 @@ stop_caddy() {
echo "==> Processing config files..."
generate_caddyfile
sync_tls_state >/dev/null
# Nothing is cached yet, so hosts left on a fallback certificate get a new ACME attempt
"$PYTHON" "${SCRIPT_DIR}/process_config.py" --reset-fallback-certificates
echo "==> Starting Caddy..."
start_caddy
if [ "$CLUSTER_NODE" != "true" ]; then
# Certificates used by cluster workers are exported as soon as they are issued or renewed
(
while true; do
sleep 60
"$PYTHON" "${SCRIPT_DIR}/process_config.py" --export-certificates || true
done
) &
fi
sleep 2
caddyfile_checksum() {
sha256sum "$CADDYFILE_PATH" 2>/dev/null | cut -d ' ' -f 1
}
APPLIED_CADDYFILE="$(caddyfile_checksum)"
echo "==> Watching ${JSON_DIR} for config changes..."
while true; do
inotifywait -qq -e close_write,moved_to,create "${JSON_DIR}/" --include '.*\.json$' 2>/dev/null || true
# The timeout also picks up changes written before the watch started or missed while reloading
inotifywait -qq -t 60 -e close_write,moved_to,create "${JSON_DIR}/" --include '.*\.json$' 2>/dev/null || true
sleep 1
echo "==> Config change detected, reprocessing..."
# The container that owns this network namespace was restarted: exit so Docker restarts Caddy and it
# joins the new namespace (otherwise it keeps serving from a namespace without published ports)
if [ -f "${JSON_DIR}/.netns_owner" ] && [ "$(cat "${JSON_DIR}/.netns_owner")" != "$(readlink /proc/self/ns/net)" ]; then
echo "Network namespace owner changed, restarting..."
stop_caddy
exit 1
fi
generate_caddyfile
if [ "$(sync_tls_state)" = "restart" ]; then
if ! kill -0 "$CADDY_PID" 2>/dev/null; then
echo "==> Caddy is not running, starting it again..."
sync_tls_state >/dev/null
start_caddy
APPLIED_CADDYFILE="$(caddyfile_checksum)"
elif [ "$(sync_tls_state)" = "restart" ]; then
# A reload keeps previously issued certificates cached in memory
echo "==> Certificate settings changed, restarting Caddy..."
stop_caddy
start_caddy
else
echo "==> Reloading Caddy..."
caddy reload --config "$CADDYFILE_PATH" --adapter caddyfile 2>/dev/null || echo "Warning: Caddy reload failed, will retry on next change."
APPLIED_CADDYFILE="$(caddyfile_checksum)"
elif [ "$(caddyfile_checksum)" != "$APPLIED_CADDYFILE" ]; then
echo "==> Config change detected, reloading Caddy..."
if caddy reload --config "$CADDYFILE_PATH" --adapter caddyfile 2>/dev/null; then
APPLIED_CADDYFILE="$(caddyfile_checksum)"
else
echo "Warning: Caddy reload failed, will retry on next change."
fi
fi
done
+171 -17
View File
@@ -36,6 +36,10 @@ TLS_STATE_PATH = os.path.join(CADDY_DATA_DIR, "wireguard_webadmin_tls_state.json
TLS_STATE_PENDING_PATH = f"{CADDYFILE_PATH}.tls_state.json"
# Set by entrypoint.sh when the generated Caddyfile fails validation
DISABLE_DNS_CHALLENGE = os.environ.get("DISABLE_DNS_CHALLENGE", "") == "1"
# Cluster nodes never request ACME certificates: they use the ones exported by the primary server
CLUSTER_NODE = os.environ.get("CLUSTER_NODE", "").lower() == "true"
NODE_CERTIFICATES_DIR = os.environ.get("NODE_CERTIFICATES_DIR", "/caddy_certificates")
CERTIFICATE_EXPORT_DIR = os.environ.get("CERTIFICATE_EXPORT_DIR", "/caddy_cert_export")
AUTH_IDENTITY_HEADERS = (
"X-Auth-User",
"X-Auth-Email",
@@ -92,12 +96,30 @@ def _safe_identifier(val: str) -> str:
return re.sub(r"[^A-Za-z0-9_]", "", str(val))
def get_node_host_tls(host_tls):
"""
TLS settings of a host on a cluster node: the certificate exported by the primary server, the internal CA
as fallback, or None when there is no certificate yet and the fallback is disabled.
"""
certificate_type = host_tls.get("certificate_type", DEFAULT_CERTIFICATE_TYPE)
certificate_fallback = host_tls.get("certificate_fallback", True) is not False
certificate_file = re.sub(r"[^a-f0-9]", "", str(host_tls.get("certificate_file", "")))
if certificate_type in ACME_CERTIFICATE_TYPES and certificate_file:
return "certificate_file", certificate_file, certificate_fallback
if certificate_type == "self_signed" or certificate_fallback:
return "self_signed", None, True
return None
def get_host_tls(host, tls_data):
"""
Return (certificate_type, dns_credential_name, certificate_fallback) for a host.
Hosts not listed use the HTTP challenge with fallback.
Hosts not listed use the HTTP challenge with fallback. On cluster nodes the second item is the
certificate file name, and None is returned for hosts that must not be served.
"""
host_tls = tls_data.get("hosts", {}).get(host.lower(), {})
if CLUSTER_NODE:
return get_node_host_tls(host_tls)
certificate_type = host_tls.get("certificate_type", DEFAULT_CERTIFICATE_TYPE)
if certificate_type not in CERTIFICATE_TYPES:
certificate_type = DEFAULT_CERTIFICATE_TYPE
@@ -117,15 +139,29 @@ def group_hosts_by_tls(hosts, tls_data):
"""Hosts sharing the same certificate settings are served by the same site block."""
groups = {}
for host in hosts:
groups.setdefault(get_host_tls(host, tls_data), []).append(host)
host_tls = get_host_tls(host, tls_data)
if host_tls is not None:
groups.setdefault(host_tls, []).append(host)
return groups.items()
def get_certificate_only_hosts(apps, tls_data):
"""Hosts the primary server only manages certificates for, because their application is served by workers."""
if CLUSTER_NODE:
return []
served_hosts = {host.lower() for app in apps for host in app.get("hosts", [])}
return [
host for host, host_tls in tls_data.get("hosts", {}).items()
if host_tls.get("certificate_only") and host not in served_hosts
]
def build_caddyfile(apps, auth_policies, routes, tls_data=None):
tls_data = tls_data or {}
policies = auth_policies.get("policies", {}) if auth_policies else {}
route_entries = routes.get("entries", {}) if routes else {}
lines = []
# Caddy refuses an empty Caddyfile, which happens on cluster nodes without applications
lines = ["# Generated by process_config.py. Do not edit.", ""]
def get_policy_type(policy_name):
if policy_name and policy_name in policies:
@@ -144,6 +180,12 @@ def build_caddyfile(apps, auth_policies, routes, tls_data=None):
lines.append("")
return
if certificate_type == "certificate_file":
certificate_path = os.path.join(NODE_CERTIFICATES_DIR, dns_credential)
lines.append(f" tls {certificate_path}.crt {certificate_path}.key")
lines.append("")
return
lines.append(" tls {")
if certificate_type == "dns_challenge":
dns_credential_data = tls_data["dns_credentials"][dns_credential]
@@ -299,6 +341,15 @@ def build_caddyfile(apps, auth_policies, routes, tls_data=None):
for (certificate_type, dns_credential, certificate_fallback), group_hosts in group_hosts_by_tls(hosts, tls_data):
emit_site_block(app, group_hosts, certificate_type, dns_credential, certificate_fallback)
certificate_only_hosts = get_certificate_only_hosts(apps, tls_data)
for (certificate_type, dns_credential, certificate_fallback), group_hosts in group_hosts_by_tls(certificate_only_hosts, tls_data):
# Keeps the certificate managed for cluster workers; the application is not published here
lines.append(f"{', '.join(_safe_value(host) for host in group_hosts)} {{")
emit_tls(certificate_type, dns_credential, certificate_fallback)
lines.append(" respond 404")
lines.append("}")
lines.append("")
return "\n".join(lines)
@@ -306,19 +357,21 @@ def build_tls_state(apps, tls_data):
"""Map each host to a fingerprint of the certificate settings used in the generated Caddyfile."""
tls_data = tls_data or {}
tls_state = {}
for app in apps:
if not app.get("hosts") or not app.get("upstream"):
hosts = [host for app in apps if app.get("upstream") for host in app.get("hosts", [])]
hosts += get_certificate_only_hosts(apps, tls_data)
for host in hosts:
host_tls = get_host_tls(host, tls_data)
if host_tls is None:
continue
for host in app["hosts"]:
certificate_type, dns_credential, certificate_fallback = get_host_tls(host, tls_data)
credential_hash = ""
if dns_credential:
credential_data = json.dumps(tls_data["dns_credentials"][dns_credential], sort_keys=True)
credential_hash = hashlib.sha256(credential_data.encode()).hexdigest()
certificate_generation = tls_data.get("hosts", {}).get(host.lower(), {}).get("certificate_generation", 0)
tls_state[host.lower()] = [
certificate_type, dns_credential or "", credential_hash, certificate_fallback, certificate_generation
]
certificate_type, certificate_reference, certificate_fallback = host_tls
credential_hash = ""
if certificate_type == "dns_challenge":
credential_data = json.dumps(tls_data["dns_credentials"][certificate_reference], sort_keys=True)
credential_hash = hashlib.sha256(credential_data.encode()).hexdigest()
certificate_generation = tls_data.get("hosts", {}).get(host.lower(), {}).get("certificate_generation", 0)
tls_state[host.lower()] = [
certificate_type, certificate_reference or "", credential_hash, certificate_fallback, certificate_generation
]
return tls_state
@@ -363,9 +416,22 @@ def sync_tls_state():
restart_required = False
if applied_state is not None:
for host, host_state in pending_state.items():
if applied_state.get(host) in (None, host_state):
previous_host_state = applied_state.get(host)
if previous_host_state == host_state or "/" in host or host in ("", ".", ".."):
continue
if "/" in host or host in ("", ".", ".."):
# A new host normally has nothing cached, unless a fallback certificate was left by an earlier use
# of the same hostname (e.g. removed from one application and added to another)
if previous_host_state is None and not os.path.isdir(
os.path.join(CERTIFICATES_DIR, INTERNAL_ISSUER_DIR_NAME, host)
):
continue
if CLUSTER_NODE:
# Certificates received from the primary server replace cached ones only after a restart
if host_state[0] == "certificate_file":
fallback_certificate_dir = os.path.join(CERTIFICATES_DIR, INTERNAL_ISSUER_DIR_NAME, host)
if os.path.isdir(fallback_certificate_dir):
shutil.rmtree(fallback_certificate_dir)
restart_required = True
continue
if host_state[0] not in ACME_CERTIFICATE_TYPES:
# A cached ACME certificate would keep being served instead of the self-signed one
@@ -386,6 +452,90 @@ def sync_tls_state():
print("restart")
def reset_fallback_certificates():
"""
Remove fallback certificates of ACME hosts before Caddy starts, so the ACME certificate is requested again
on every start instead of only on the next renewal. If the request fails, the fallback is issued again.
"""
pending_state = _read_json(TLS_STATE_PENDING_PATH) or {}
for host, host_state in pending_state.items():
if host_state[0] not in ACME_CERTIFICATE_TYPES or "/" in host or host in ("", ".", ".."):
continue
fallback_certificate_dir = os.path.join(CERTIFICATES_DIR, INTERNAL_ISSUER_DIR_NAME, host)
if os.path.isdir(fallback_certificate_dir):
shutil.rmtree(fallback_certificate_dir)
print(f"Removed fallback certificate for {host}", file=sys.stderr)
def find_acme_certificate(host):
"""Return (certificate_path, key_path) of the newest ACME certificate stored by Caddy for a host."""
candidates = []
if os.path.isdir(CERTIFICATES_DIR):
for issuer_dir_name in os.listdir(CERTIFICATES_DIR):
if issuer_dir_name == INTERNAL_ISSUER_DIR_NAME:
continue
certificate_path = os.path.join(CERTIFICATES_DIR, issuer_dir_name, host, f"{host}.crt")
key_path = os.path.join(CERTIFICATES_DIR, issuer_dir_name, host, f"{host}.key")
if os.path.isfile(certificate_path) and os.path.isfile(key_path):
candidates.append((os.path.getmtime(certificate_path), certificate_path, key_path))
if not candidates:
return None, None
newest = max(candidates)
return newest[1], newest[2]
def _copy_if_changed(source_path, destination_path):
with open(source_path, "r", encoding="utf-8") as source_file:
content = source_file.read()
if _read_text(destination_path) == content:
return False
temp_path = f"{destination_path}.tmp"
with open(temp_path, "w", encoding="utf-8") as destination_file:
destination_file.write(content)
os.chmod(temp_path, 0o600)
os.replace(temp_path, destination_path)
return True
def _read_text(filepath):
try:
with open(filepath, "r", encoding="utf-8") as text_file:
return text_file.read()
except OSError:
return None
def export_certificates():
"""
Copy the ACME certificates of hosts served by cluster workers to CERTIFICATE_EXPORT_DIR, where Django reads
them to distribute to the workers. Fallback certificates from the internal CA are never exported.
"""
if CLUSTER_NODE or not os.path.isdir(CERTIFICATE_EXPORT_DIR):
return
tls_data = load_json("tls.json") or {}
export_hosts = {
host for host, host_tls in tls_data.get("hosts", {}).items()
if host_tls.get("export_certificate") and "/" not in host and host not in ("", ".", "..")
}
for host in sorted(export_hosts):
certificate_path, key_path = find_acme_certificate(host)
if not certificate_path:
continue
host_export_dir = os.path.join(CERTIFICATE_EXPORT_DIR, host)
os.makedirs(host_export_dir, exist_ok=True)
key_changed = _copy_if_changed(key_path, os.path.join(host_export_dir, "certificate.key"))
certificate_changed = _copy_if_changed(certificate_path, os.path.join(host_export_dir, "certificate.crt"))
if certificate_changed or key_changed:
print(f"Exported certificate for {host}")
for host in os.listdir(CERTIFICATE_EXPORT_DIR):
host_export_dir = os.path.join(CERTIFICATE_EXPORT_DIR, host)
if host not in export_hosts and os.path.isdir(host_export_dir):
shutil.rmtree(host_export_dir)
print(f"Removed exported certificate for {host}")
def main():
apps = collect_all_applications()
auth_policies = load_json("auth_policies.json")
@@ -403,5 +553,9 @@ def main():
if __name__ == "__main__":
if "--sync-tls-state" in sys.argv:
sync_tls_state()
elif "--export-certificates" in sys.argv:
export_certificates()
elif "--reset-fallback-certificates" in sys.argv:
reset_fallback_certificates()
else:
main()
@@ -12,6 +12,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
inetutils-traceroute \
procps \
curl \
openssl \
nano \
&& rm -rf /var/lib/apt/lists/*
+394 -312
View File
@@ -1,350 +1,432 @@
import glob
import json
import logging
import os
import re
import socket
import ssl
import subprocess
import tempfile
import time
from typing import Dict, Any
from datetime import datetime, timezone
import requests
# Configure logging
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s'
)
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
# Constants
MASTER_SERVER_ADDRESS = os.environ.get('MASTER_SERVER_ADDRESS')
TOKEN = os.environ.get('TOKEN')
MASTER_SERVER_ADDRESS = os.environ.get('MASTER_SERVER_ADDRESS', '')
TOKEN = os.environ.get('TOKEN', '')
WORKER_VERSION = 12
REQUEST_TIMEOUT = 30
MANIFEST_INTERVAL = 60
DEFAULT_STATS_INTERVAL = 60
WIREGUARD_DIR = '/etc/wireguard'
FIREWALL_SCRIPT = os.path.join(WIREGUARD_DIR, 'wg-firewall.sh')
DNS_DIR = '/etc/dnsmasq'
WORKER_VERSION = 11
REQUEST_TIMEOUT = 20
class ClusterWorker:
def __init__(self):
self.config_version = 0
self.dns_version = self.get_local_dns_version()
self.should_run = True
self.session = requests.Session()
if not MASTER_SERVER_ADDRESS or not TOKEN:
logger.error("MASTER_SERVER_ADDRESS or TOKEN not set")
self.should_run = False
self.base_url = f"https://{MASTER_SERVER_ADDRESS}/api/cluster"
# Validate URL scheme
if not MASTER_SERVER_ADDRESS.startswith(('http://', 'https://')):
self.base_url = f"https://{MASTER_SERVER_ADDRESS}/api/cluster"
else:
self.base_url = f"{MASTER_SERVER_ADDRESS}/api/cluster"
DNS_MAIN_CONFIG = 'wireguard_webadmin_dns.conf'
# Only mounted when the node runs the App Gateway (cluster_node-gateway.yml)
GATEWAY_DIR = '/caddy_json_export'
CERTIFICATES_DIR = '/caddy_certificates'
GATEWAY_FILES = ('applications.json', 'auth_policies.json', 'routes.json', 'tls.json')
CERTIFICATE_FILE_PATTERN = re.compile(r'^[a-f0-9]{64}\.(crt|key)$')
CADDY_ADMIN_URL = 'http://127.0.0.1:2019/config/'
AUTH_GATEWAY_URL = 'http://cluster-node-auth-gateway:9091/'
INTERNAL_CA_NAME = 'Caddy Local Authority'
FIREWALL_CHAINS = (
('nat', 'WGWADM_POSTROUTING'),
('nat', 'WGWADM_PREROUTING'),
('filter', 'WGWADM_FORWARD'),
('filter', 'WGWADM_ROUTE_POLICY'),
)
def func_process_wireguard_status(self) -> Dict[str, Any]:
command = "wg show all dump"
process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
stdout, stderr = process.communicate()
if process.returncode != 0:
return {"message": stderr, "status": "error"}
data: Dict[str, Dict[str, Dict[str, Any]]] = {}
# wg dump format is tab-separated.
# There are two kinds of lines:
# - Interface line: interface \t private_key \t public_key \t listen_port \t fwmark
# - Peer line: interface \t peer_public_key \t preshared_key \t endpoint \t allowed_ips \t latest_handshake \t transfer_rx \t transfer_tx \t persistent_keepalive
for line in stdout.strip().split("\n"):
parts = line.split("\t")
if len(parts) < 5:
continue
interface = parts[0]
# Peer lines are expected to have at least 9 fields
if len(parts) < 9:
# interface line, ignore
continue
peer_public_key = parts[1]
endpoint = parts[3]
allowed_ips_raw = parts[4]
latest_handshake = parts[5]
transfer_rx = parts[6]
transfer_tx = parts[7]
if interface not in data:
data[interface] = {}
data[interface][peer_public_key] = {
"allowed-ips": [ip for ip in allowed_ips_raw.split(",") if ip] if allowed_ips_raw else [],
"latest-handshakes": latest_handshake or "",
"transfer": {"tx": int(transfer_tx or 0), "rx": int(transfer_rx or 0)},
"endpoints": endpoint or "",
}
return data
class SectionError(Exception):
pass
def cleanup_wireguard(self):
logger.info("Cleaning up WireGuard configuration...")
# Stop all wireguard interfaces
try:
files = glob.glob(f"{WIREGUARD_DIR}/*.conf")
for f in files:
interface = os.path.basename(f).replace('.conf', '')
subprocess.run(['wg-quick', 'down', interface], capture_output=True)
except Exception as e:
logger.error(f"Error stopping wireguard interfaces: {e}")
def run_command(command, check=True):
result = subprocess.run(command, capture_output=True, text=True)
if check and result.returncode != 0:
raise SectionError(f"{' '.join(command)}: {(result.stderr or result.stdout).strip()}")
return result
# Remove files
try:
for f in glob.glob(f"{WIREGUARD_DIR}/*"):
os.remove(f)
except Exception as e:
logger.error(f"Error cleaning directory: {e}")
try:
subprocess.run(['iptables', '-t', 'nat', '-F', 'WGWADM_POSTROUTING'], capture_output=True)
subprocess.run(['iptables', '-t', 'nat', '-F', 'WGWADM_PREROUTING'], capture_output=True)
subprocess.run(['iptables', '-t', 'filter', '-F', 'WGWADM_FORWARD'], capture_output=True)
except Exception as e:
logger.error(f"Error flushing firewall: {e}")
def interface_is_up(interface):
return run_command(['wg', 'show', interface], check=False).returncode == 0
def get_local_dns_version(self):
try:
version_file = os.path.join(DNS_DIR, 'config_version.conf')
if os.path.exists(version_file):
with open(version_file, 'r') as f:
for line in f:
if line.startswith('DNS_VERSION='):
return int(line.strip().split('=')[1])
except Exception as e:
logger.error(f"Error reading DNS version: {e}")
return 0
def get_status(self):
params = {
'token': TOKEN,
'worker_config_version': self.config_version,
'worker_dns_version': self.dns_version,
'worker_version': WORKER_VERSION
def interface_section(config):
"""Return the [Interface] part of a WireGuard config, used to decide between syncconf and restart."""
return config.split('[Peer]', 1)[0].strip()
def read_file(path):
try:
with open(path, 'r', encoding='utf-8') as config_file:
return config_file.read()
except FileNotFoundError:
return None
def write_file(path, content, mode=0o600, in_place=False):
if in_place:
with open(path, 'w', encoding='utf-8') as config_file:
config_file.write(content)
os.chmod(path, mode)
return
temp_path = f'{path}.tmp'
with open(temp_path, 'w', encoding='utf-8') as config_file:
config_file.write(content)
os.chmod(temp_path, mode)
os.replace(temp_path, path)
def wireguard_status():
result = run_command(['wg', 'show', 'all', 'dump'], check=False)
if result.returncode != 0:
return {}
data = {}
# Peer lines: interface, public_key, preshared_key, endpoint, allowed_ips, latest_handshake, rx, tx, keepalive
for line in result.stdout.strip().splitlines():
parts = line.split('\t')
if len(parts) < 9:
continue
interface, peer_public_key, endpoint, allowed_ips = parts[0], parts[1], parts[3], parts[4]
data.setdefault(interface, {})[peer_public_key] = {
'allowed-ips': [ip for ip in allowed_ips.split(',') if ip] if allowed_ips else [],
'latest-handshakes': parts[5] or '',
'transfer': {'tx': int(parts[7] or 0), 'rx': int(parts[6] or 0)},
'endpoints': endpoint or '',
}
logger.info(f"Requesting status from Master... (Config Version: {self.config_version}, DNS Version: {self.dns_version}, Worker Version: {WORKER_VERSION})")
try:
response = self.session.get(f"{self.base_url}/status/", params=params, timeout=REQUEST_TIMEOUT)
logger.info(f"Status response received. HTTP Code: {response.status_code}")
return response
except requests.RequestException as e:
logger.error(f"Connection error while getting status: {e}")
return None
return data
def download_configs(self):
params = {
'token': TOKEN,
'worker_config_version': self.config_version,
'worker_dns_version': self.dns_version,
'worker_version': WORKER_VERSION
def gateway_enabled():
return os.path.isdir(GATEWAY_DIR) and os.path.isdir(CERTIFICATES_DIR)
def http_service_is_running(url):
try:
requests.get(url, timeout=3)
return True
except requests.RequestException:
return False
def _parse_not_after(value):
return datetime.strptime(value, '%b %d %H:%M:%S %Y %Z').replace(tzinfo=timezone.utc).isoformat()
def _fetch_certificate(hostname, verify):
if verify:
context = ssl.create_default_context()
else:
context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
with socket.create_connection(('127.0.0.1', 443), timeout=3) as raw_socket:
with context.wrap_socket(raw_socket, server_hostname=hostname) as tls_socket:
return tls_socket.getpeercert() if verify else tls_socket.getpeercert(binary_form=True)
def inspect_certificate(hostname):
"""Same checks as app_gateway/certificate_status.py on the primary server, against the local Caddy."""
try:
certificate = _fetch_certificate(hostname, verify=True)
issuer = dict(field[0] for field in certificate.get('issuer', ()))
return {
'status': 'trusted',
'issuer': issuer.get('organizationName') or issuer.get('commonName') or '',
'not_after': _parse_not_after(certificate['notAfter']),
}
try:
response = self.session.get(f"{self.base_url}/worker/get_config_files/", params=params, timeout=REQUEST_TIMEOUT)
if response.status_code == 200:
data = response.json()
if data.get('status') == 'success':
return data
return None
except requests.RequestException as e:
logger.error(f"Error downloading configs: {e}")
return None
except ssl.SSLCertVerificationError:
pass
except (OSError, ValueError):
return {'status': 'unavailable', 'issuer': '', 'not_after': None}
def download_dns_config(self):
params = {
'token': TOKEN,
'worker_config_version': self.config_version,
'worker_dns_version': self.dns_version,
'worker_version': WORKER_VERSION
}
try:
logger.info(f"Downloading DNS config... (Current Version: {self.dns_version})")
response = self.session.get(f"{self.base_url}/worker/get_dnsmasq_config/", params=params, timeout=REQUEST_TIMEOUT)
if response.status_code == 200:
tar_path = 'dnsmasq_config.tar.gz'
with open(tar_path, 'wb') as f:
f.write(response.content)
logger.info("Extracting DNS config...")
if not os.path.exists(DNS_DIR):
os.makedirs(DNS_DIR, exist_ok=True)
subprocess.run(['tar', 'xvfz', tar_path, '-C', DNS_DIR], check=True, capture_output=True)
if os.path.exists(tar_path):
os.remove(tar_path)
# Update version
self.dns_version = self.get_local_dns_version()
logger.info(f"DNS config updated (New Version: {self.dns_version})")
return True
else:
logger.error(f"Failed to download DNS config. Status: {response.status_code}")
return False
except Exception as e:
logger.error(f"Error updating DNS config: {e}")
return False
try:
certificate_der = _fetch_certificate(hostname, verify=False)
result = subprocess.run(
['openssl', 'x509', '-inform', 'DER', '-noout', '-issuer', '-enddate', '-nameopt', 'RFC2253'],
input=certificate_der, capture_output=True, timeout=5,
)
except (OSError, ValueError, subprocess.SubprocessError):
return {'status': 'unavailable', 'issuer': '', 'not_after': None}
issuer = ''
not_after = None
for line in result.stdout.decode(errors='replace').splitlines():
if line.startswith('issuer='):
parts = dict(part.split('=', 1) for part in line[len('issuer='):].split(',') if '=' in part)
issuer = INTERNAL_CA_NAME if INTERNAL_CA_NAME in line else parts.get('O') or parts.get('CN') or ''
elif line.startswith('notAfter='):
not_after = _parse_not_after(line[len('notAfter='):])
return {'status': 'untrusted', 'issuer': issuer, 'not_after': not_after}
def send_ping(self):
params = {
'token': TOKEN,
'worker_config_version': self.config_version,
'worker_dns_version': self.dns_version,
'worker_version': WORKER_VERSION
}
try:
logger.info("Sending ping to Master...")
self.session.get(f"{self.base_url}/worker/ping/", params=params, timeout=REQUEST_TIMEOUT)
except Exception as e:
logger.error(f"Error sending ping: {e}")
def apply_configs(self, data):
logger.info("Applying new configurations...")
files = data.get('files', {})
cluster_settings = data.get('cluster_settings', {})
new_config_version = cluster_settings.get('config_version', 0)
# 1. Stop existing interfaces
self.cleanup_wireguard()
# 2. Write new files
for filename, content in files.items():
if filename == 'wg-firewall.sh' and isinstance(content, str):
content = content.replace('wireguard-webadmin-dns', 'cluster-node-dns')
filepath = os.path.join(WIREGUARD_DIR, filename)
with open(filepath, 'w') as f:
f.write(content)
if filename == 'wg-firewall.sh':
os.chmod(filepath, 0o755)
# Start interfaces
conf_files = glob.glob(f"{WIREGUARD_DIR}/*.conf")
for conf in conf_files:
interface = os.path.basename(conf).replace('.conf', '')
logger.info(f"Starting WireGuard interface: {interface}")
def dns_is_running():
# dnsmasq always listens on loopback when it is restricted to specific interfaces. The DNS container pauses
# for a few seconds while reloading, so a missing answer is retried before reporting it as stopped.
query = b'\x12\x34\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x09localhost\x00\x00\x01\x00\x01'
for attempt in range(4):
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as dns_socket:
dns_socket.settimeout(2)
try:
subprocess.run(['wg-quick', 'up', interface], check=True, capture_output=True)
except subprocess.CalledProcessError as e:
logger.error(f"Failed to start {interface}: {e.stderr.decode()}")
dns_socket.sendto(query, ('127.0.0.1', 53))
dns_socket.recvfrom(512)
return True
except OSError:
pass
time.sleep(1)
return False
# 4. Update config version
self.config_version = new_config_version
logger.info(f"Configuration updated to version {self.config_version}")
class ClusterNodeAgent:
def __init__(self):
self.base_url = f"{MASTER_SERVER_ADDRESS.rstrip('/')}/api/cluster/v2"
self.session = requests.Session()
self.session.headers.update({
'Authorization': f'Bearer {TOKEN}',
'X-Worker-Version': str(WORKER_VERSION),
})
# Nothing is persisted: after a restart interfaces and firewall rules are gone, so every section is applied again
self.applied_hashes = {}
self.stats_interval = DEFAULT_STATS_INTERVAL
self.wireguard_active = False
def request(self, method, path, **kwargs):
try:
return self.session.request(method, f'{self.base_url}/{path}', timeout=REQUEST_TIMEOUT, **kwargs)
except requests.RequestException as error:
logger.error('Request to %s failed: %s', path, error)
return None
# Section handlers
def apply_firewall(self, section):
content = section['files'].get('wg-firewall.sh')
if content is None:
raise SectionError('wg-firewall.sh missing from section')
write_file(FIREWALL_SCRIPT, content, mode=0o700)
run_command(['bash', FIREWALL_SCRIPT])
def apply_wireguard(self, section):
desired_files = {
filename: content for filename, content in section['files'].items()
if filename.startswith('wg') and filename.endswith('.conf') and '/' not in filename
}
errors = []
for config_path in glob.glob(os.path.join(WIREGUARD_DIR, 'wg[0-9]*.conf')):
filename = os.path.basename(config_path)
if filename in desired_files:
continue
interface = filename[:-len('.conf')]
if interface_is_up(interface):
run_command(['wg-quick', 'down', interface], check=False)
os.remove(config_path)
logger.info('Removed WireGuard interface %s', interface)
for filename, content in sorted(desired_files.items()):
interface = filename[:-len('.conf')]
config_path = os.path.join(WIREGUARD_DIR, filename)
previous_content = read_file(config_path)
write_file(config_path, content)
try:
if not interface_is_up(interface):
run_command(['wg-quick', 'up', interface])
logger.info('Started WireGuard interface %s', interface)
elif previous_content is None or interface_section(previous_content) != interface_section(content):
run_command(['wg-quick', 'down', interface], check=False)
run_command(['wg-quick', 'up', interface])
logger.info('Restarted WireGuard interface %s', interface)
elif previous_content != content:
self.sync_peers(interface, config_path)
logger.info('Synchronized peers of WireGuard interface %s', interface)
except SectionError as error:
errors.append(str(error))
self.wireguard_active = True
if errors:
raise SectionError('; '.join(errors))
@staticmethod
def sync_peers(interface, config_path):
stripped_config = run_command(['wg-quick', 'strip', config_path]).stdout
with tempfile.NamedTemporaryFile('w', suffix='.conf', delete=False) as stripped_file:
stripped_file.write(stripped_config)
try:
run_command(['wg', 'syncconf', interface, stripped_file.name])
finally:
os.remove(stripped_file.name)
def apply_dns(self, section):
files = section['files']
if DNS_MAIN_CONFIG not in files:
raise SectionError(f'{DNS_MAIN_CONFIG} missing from section')
os.makedirs(DNS_DIR, exist_ok=True)
for filename, content in files.items():
if filename != DNS_MAIN_CONFIG and '/' not in filename:
write_file(os.path.join(DNS_DIR, filename), content, mode=0o644)
for config_path in glob.glob(os.path.join(DNS_DIR, '*.conf')):
if os.path.basename(config_path) not in files:
os.remove(config_path)
# Written last and in place: the DNS container watches this file with inotify to reload
write_file(os.path.join(DNS_DIR, DNS_MAIN_CONFIG), files[DNS_MAIN_CONFIG], mode=0o644, in_place=True)
def apply_settings(self, section):
self.stats_interval = int(section['values'].get('stats_sync_interval', DEFAULT_STATS_INTERVAL))
def apply_certificates(self, section):
if not gateway_enabled():
return
files = {
filename: content for filename, content in section['files'].items()
if CERTIFICATE_FILE_PATTERN.match(filename)
}
for filename, content in files.items():
certificate_path = os.path.join(CERTIFICATES_DIR, filename)
if read_file(certificate_path) != content:
write_file(certificate_path, content, mode=0o600)
for certificate_path in glob.glob(os.path.join(CERTIFICATES_DIR, '*')):
if os.path.basename(certificate_path) not in files:
os.remove(certificate_path)
def apply_gateway(self, section):
if not gateway_enabled():
return
for filename in GATEWAY_FILES:
if filename not in section['files']:
raise SectionError(f'{filename} missing from section')
for filename in GATEWAY_FILES:
write_file(os.path.join(GATEWAY_DIR, filename), section['files'][filename], mode=0o644)
SECTION_HANDLERS = {
'firewall': apply_firewall,
'wireguard': apply_wireguard,
'dns': apply_dns,
'settings': apply_settings,
'certificates': apply_certificates,
'gateway': apply_gateway,
}
# Main loop
def shutdown_wireguard(self):
if not self.wireguard_active:
return
logger.warning('Shutting down WireGuard interfaces and firewall rules')
for config_path in glob.glob(os.path.join(WIREGUARD_DIR, 'wg[0-9]*.conf')):
run_command(['wg-quick', 'down', os.path.basename(config_path)[:-len('.conf')]], check=False)
for table, chain in FIREWALL_CHAINS:
run_command(['iptables', '-t', table, '-F', chain], check=False)
self.applied_hashes.pop('wireguard', None)
self.applied_hashes.pop('firewall', None)
self.wireguard_active = False
def sync_configuration(self):
response = self.request('GET', 'manifest/')
if response is None:
return True
if response.status_code == 403:
logger.error('Token rejected by the primary server. Stopping.')
self.shutdown_wireguard()
return False
if response.status_code == 400:
logger.warning('Primary server refused this worker: %s', response.json().get('message', ''))
self.shutdown_wireguard()
return True
if response.status_code != 200:
logger.error('Unexpected manifest response: HTTP %s', response.status_code)
return True
manifest = response.json().get('sections', {})
reports = {}
for section_name, section_hash in manifest.items():
if section_name not in self.SECTION_HANDLERS or self.applied_hashes.get(section_name) == section_hash:
continue
section_response = self.request('GET', f'section/{section_name}/')
if section_response is None or section_response.status_code != 200:
continue
section = section_response.json()
try:
self.SECTION_HANDLERS[section_name](self, section)
self.applied_hashes[section_name] = section['hash']
reports[section_name] = {'hash': section['hash'], 'status': 'ok'}
logger.info('Section %s applied', section_name)
except (SectionError, OSError, ValueError) as error:
# Forget the last applied content: the next version must be applied and reported even if it
# is identical to the one that was applied before the failure
self.applied_hashes.pop(section_name, None)
reports[section_name] = {'hash': section['hash'], 'status': 'error', 'message': str(error)}
logger.error('Section %s failed: %s', section_name, error)
if reports:
# Services are reported with the periodic stats: right after a DNS change dnsmasq is still reloading
self.request('POST', 'report/', json={'sections': reports})
return True
@staticmethod
def services_status():
services = {'dnsmasq': 'running' if dns_is_running() else 'stopped'}
if gateway_enabled():
services['caddy'] = 'running' if http_service_is_running(CADDY_ADMIN_URL) else 'stopped'
services['auth_gateway'] = 'running' if http_service_is_running(AUTH_GATEWAY_URL) else 'stopped'
return services
@staticmethod
def certificates_status():
try:
with open(os.path.join(GATEWAY_DIR, 'tls.json'), 'r', encoding='utf-8') as tls_file:
hosts = json.load(tls_file).get('hosts', {})
except (OSError, ValueError):
return {}
return {hostname: inspect_certificate(hostname) for hostname in sorted(hosts)}
def send_stats(self):
try:
stats = self.func_process_wireguard_status()
params = {
'token': TOKEN,
'worker_config_version': self.config_version,
'worker_dns_version': self.dns_version,
'worker_version': WORKER_VERSION
}
logger.info("Sending WireGuard stats to Master...")
response = self.session.post(f"{self.base_url}/worker/submit_wireguard_stats/", json=stats, params=params, timeout=REQUEST_TIMEOUT)
if response.status_code == 200:
logger.info("Stats sent successfully.")
return True
else:
logger.error(f"Failed to send stats. Status: {response.status_code}")
return False
except Exception as e:
logger.error(f"Error sending stats: {e}")
return False
report = {
'wireguard_status': wireguard_status(),
'services': self.services_status(),
}
if gateway_enabled():
report['certificates'] = self.certificates_status()
self.request('POST', 'report/', json=report)
@staticmethod
def publish_network_namespace():
# Containers sharing this network namespace (DNS, Caddy) restart themselves when it changes
network_namespace = os.readlink('/proc/self/ns/net')
for directory in (DNS_DIR, GATEWAY_DIR):
if os.path.isdir(directory):
write_file(os.path.join(directory, '.netns_owner'), network_namespace + '\n', mode=0o644)
def run(self):
if not self.should_run:
return
logger.info("Cluster Worker starting...")
# Initial cleanup
self.cleanup_wireguard()
last_config_check = 0
last_stats_send = 0
stats_sync_interval = 60 # Default initial value
logger.info('Cluster node agent %s starting', WORKER_VERSION)
self.publish_network_namespace()
self.wireguard_active = True
self.shutdown_wireguard()
last_manifest = 0
last_stats = 0
while True:
current_time = time.time()
# Check Config (Fixed 60s interval)
if current_time - last_config_check >= 60:
try:
response = self.get_status()
last_config_check = time.time()
if response is not None:
if response.status_code == 403:
logger.error("Received 403 Forbidden (Token invalid/deleted). Deactivating WireGuard and stopping requests permanently.")
self.cleanup_wireguard()
self.config_version = 0
self.should_run = False
break
if response.status_code == 400:
logger.warning("Received 400 Bad Request (Worker suspended or error). Deactivating WireGuard and Firewall, but will keep retrying...")
self.cleanup_wireguard()
self.config_version = 0
if response.status_code == 200:
data = response.json()
# Update stats interval from master settings
cluster_settings = data.get('cluster_settings', {})
stats_sync_interval = cluster_settings.get('stats_sync_interval', 60)
remote_config_version = cluster_settings.get('config_version', 0)
# Check WireGuard Config
updated = False
if int(remote_config_version) != self.config_version:
logger.info(f"Config version mismatch (Local: {self.config_version}, Remote: {remote_config_version}). Updating...")
config_data = self.download_configs()
if config_data:
self.apply_configs(config_data)
self.send_ping()
updated = True
else:
logger.error("Failed to download config files.")
# Check DNS Config
remote_dns_version = int(cluster_settings.get('dns_version', 0))
if not updated and remote_dns_version != self.dns_version:
logger.info(f"DNS version mismatch (Local: {self.dns_version}, Remote: {remote_dns_version}). Updating...")
if self.download_dns_config():
self.send_ping()
updated = True
if not updated and int(remote_config_version) == self.config_version and remote_dns_version == self.dns_version:
logger.info(f"No changes detected. Configuration is up to date (WG: {self.config_version}, DNS: {self.dns_version}).")
except Exception as e:
logger.error(f"Unexpected error in config check loop: {e}")
# Check Stats Sync
if current_time - last_stats_send >= stats_sync_interval:
now = time.monotonic()
if now - last_manifest >= MANIFEST_INTERVAL or not last_manifest:
last_manifest = now
if not self.sync_configuration():
break
if now - last_stats >= self.stats_interval:
last_stats = now
self.send_stats()
last_stats_send = time.time()
# Sleep briefly to be responsive but not busy loop
time.sleep(1)
# Final loop state if 403 was received
while not self.should_run:
logger.info("Worker disabled due to 403 Forbidden. WireGuard is off.")
time.sleep(60)
while True:
time.sleep(3600)
if __name__ == "__main__":
worker = ClusterWorker()
worker.run()
if __name__ == '__main__':
if not MASTER_SERVER_ADDRESS or not TOKEN:
logger.error('MASTER_SERVER_ADDRESS and TOKEN must be set')
else:
ClusterNodeAgent().run()
+1 -1
View File
@@ -1,7 +1,7 @@
#!/bin/bash
set -e
if [[ "$COMPOSE_VERSION" != "c1b" ]]; then
if [[ "$COMPOSE_VERSION" != "c2a" ]]; then
echo "ERROR: Please upgrade your docker compose file. Exiting."
exit 1
fi
+17
View File
@@ -34,7 +34,24 @@ handle_sigint() {
exit 0
}
# Written by cluster nodes, where this container shares the network namespace of the node. When the node
# container is restarted it gets a new namespace, so this container must restart to join it again.
NETNS_OWNER_FILE="/etc/dnsmasq/.netns_owner"
watch_network_namespace() {
while sleep 30; do
if [ -f "$NETNS_OWNER_FILE" ] && [ "$(cat "$NETNS_OWNER_FILE")" != "$(readlink /proc/self/ns/net)" ]; then
echo "Network namespace of the cluster node changed, restarting..."
kill -TERM 1
pkill inotifywait
return
fi
done
}
trap handle_sigint SIGINT
trap 'pkill dnsmasq; exit 1' SIGTERM
create_default_config
watch_network_namespace &
start_dnsmasq