mirror of
https://github.com/eduardogsilva/wireguard_webadmin.git
synced 2026-11-22 04:43:18 +01:00
Rework cluster into v2: section-based worker sync and App Gateway on workers
This commit is contained in:
@@ -12,6 +12,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
inetutils-traceroute \
|
||||
procps \
|
||||
curl \
|
||||
openssl \
|
||||
nano \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
|
||||
@@ -1,350 +1,432 @@
|
||||
import glob
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import ssl
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from typing import Dict, Any
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import requests
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Constants
|
||||
MASTER_SERVER_ADDRESS = os.environ.get('MASTER_SERVER_ADDRESS')
|
||||
TOKEN = os.environ.get('TOKEN')
|
||||
MASTER_SERVER_ADDRESS = os.environ.get('MASTER_SERVER_ADDRESS', '')
|
||||
TOKEN = os.environ.get('TOKEN', '')
|
||||
WORKER_VERSION = 12
|
||||
REQUEST_TIMEOUT = 30
|
||||
MANIFEST_INTERVAL = 60
|
||||
DEFAULT_STATS_INTERVAL = 60
|
||||
|
||||
WIREGUARD_DIR = '/etc/wireguard'
|
||||
FIREWALL_SCRIPT = os.path.join(WIREGUARD_DIR, 'wg-firewall.sh')
|
||||
DNS_DIR = '/etc/dnsmasq'
|
||||
WORKER_VERSION = 11
|
||||
REQUEST_TIMEOUT = 20
|
||||
|
||||
class ClusterWorker:
|
||||
def __init__(self):
|
||||
self.config_version = 0
|
||||
self.dns_version = self.get_local_dns_version()
|
||||
self.should_run = True
|
||||
self.session = requests.Session()
|
||||
|
||||
if not MASTER_SERVER_ADDRESS or not TOKEN:
|
||||
logger.error("MASTER_SERVER_ADDRESS or TOKEN not set")
|
||||
self.should_run = False
|
||||
|
||||
self.base_url = f"https://{MASTER_SERVER_ADDRESS}/api/cluster"
|
||||
# Validate URL scheme
|
||||
if not MASTER_SERVER_ADDRESS.startswith(('http://', 'https://')):
|
||||
self.base_url = f"https://{MASTER_SERVER_ADDRESS}/api/cluster"
|
||||
else:
|
||||
self.base_url = f"{MASTER_SERVER_ADDRESS}/api/cluster"
|
||||
DNS_MAIN_CONFIG = 'wireguard_webadmin_dns.conf'
|
||||
# Only mounted when the node runs the App Gateway (cluster_node-gateway.yml)
|
||||
GATEWAY_DIR = '/caddy_json_export'
|
||||
CERTIFICATES_DIR = '/caddy_certificates'
|
||||
GATEWAY_FILES = ('applications.json', 'auth_policies.json', 'routes.json', 'tls.json')
|
||||
CERTIFICATE_FILE_PATTERN = re.compile(r'^[a-f0-9]{64}\.(crt|key)$')
|
||||
CADDY_ADMIN_URL = 'http://127.0.0.1:2019/config/'
|
||||
AUTH_GATEWAY_URL = 'http://cluster-node-auth-gateway:9091/'
|
||||
INTERNAL_CA_NAME = 'Caddy Local Authority'
|
||||
FIREWALL_CHAINS = (
|
||||
('nat', 'WGWADM_POSTROUTING'),
|
||||
('nat', 'WGWADM_PREROUTING'),
|
||||
('filter', 'WGWADM_FORWARD'),
|
||||
('filter', 'WGWADM_ROUTE_POLICY'),
|
||||
)
|
||||
|
||||
|
||||
def func_process_wireguard_status(self) -> Dict[str, Any]:
|
||||
command = "wg show all dump"
|
||||
|
||||
process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
stdout, stderr = process.communicate()
|
||||
|
||||
if process.returncode != 0:
|
||||
return {"message": stderr, "status": "error"}
|
||||
|
||||
data: Dict[str, Dict[str, Dict[str, Any]]] = {}
|
||||
|
||||
# wg dump format is tab-separated.
|
||||
# There are two kinds of lines:
|
||||
# - Interface line: interface \t private_key \t public_key \t listen_port \t fwmark
|
||||
# - Peer line: interface \t peer_public_key \t preshared_key \t endpoint \t allowed_ips \t latest_handshake \t transfer_rx \t transfer_tx \t persistent_keepalive
|
||||
for line in stdout.strip().split("\n"):
|
||||
parts = line.split("\t")
|
||||
if len(parts) < 5:
|
||||
continue
|
||||
|
||||
interface = parts[0]
|
||||
|
||||
# Peer lines are expected to have at least 9 fields
|
||||
if len(parts) < 9:
|
||||
# interface line, ignore
|
||||
continue
|
||||
|
||||
peer_public_key = parts[1]
|
||||
endpoint = parts[3]
|
||||
allowed_ips_raw = parts[4]
|
||||
latest_handshake = parts[5]
|
||||
transfer_rx = parts[6]
|
||||
transfer_tx = parts[7]
|
||||
|
||||
if interface not in data:
|
||||
data[interface] = {}
|
||||
|
||||
data[interface][peer_public_key] = {
|
||||
"allowed-ips": [ip for ip in allowed_ips_raw.split(",") if ip] if allowed_ips_raw else [],
|
||||
"latest-handshakes": latest_handshake or "",
|
||||
"transfer": {"tx": int(transfer_tx or 0), "rx": int(transfer_rx or 0)},
|
||||
"endpoints": endpoint or "",
|
||||
}
|
||||
|
||||
return data
|
||||
class SectionError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def cleanup_wireguard(self):
|
||||
logger.info("Cleaning up WireGuard configuration...")
|
||||
# Stop all wireguard interfaces
|
||||
try:
|
||||
files = glob.glob(f"{WIREGUARD_DIR}/*.conf")
|
||||
for f in files:
|
||||
interface = os.path.basename(f).replace('.conf', '')
|
||||
subprocess.run(['wg-quick', 'down', interface], capture_output=True)
|
||||
except Exception as e:
|
||||
logger.error(f"Error stopping wireguard interfaces: {e}")
|
||||
def run_command(command, check=True):
|
||||
result = subprocess.run(command, capture_output=True, text=True)
|
||||
if check and result.returncode != 0:
|
||||
raise SectionError(f"{' '.join(command)}: {(result.stderr or result.stdout).strip()}")
|
||||
return result
|
||||
|
||||
# Remove files
|
||||
try:
|
||||
for f in glob.glob(f"{WIREGUARD_DIR}/*"):
|
||||
os.remove(f)
|
||||
except Exception as e:
|
||||
logger.error(f"Error cleaning directory: {e}")
|
||||
|
||||
try:
|
||||
subprocess.run(['iptables', '-t', 'nat', '-F', 'WGWADM_POSTROUTING'], capture_output=True)
|
||||
subprocess.run(['iptables', '-t', 'nat', '-F', 'WGWADM_PREROUTING'], capture_output=True)
|
||||
subprocess.run(['iptables', '-t', 'filter', '-F', 'WGWADM_FORWARD'], capture_output=True)
|
||||
except Exception as e:
|
||||
logger.error(f"Error flushing firewall: {e}")
|
||||
def interface_is_up(interface):
|
||||
return run_command(['wg', 'show', interface], check=False).returncode == 0
|
||||
|
||||
def get_local_dns_version(self):
|
||||
try:
|
||||
version_file = os.path.join(DNS_DIR, 'config_version.conf')
|
||||
if os.path.exists(version_file):
|
||||
with open(version_file, 'r') as f:
|
||||
for line in f:
|
||||
if line.startswith('DNS_VERSION='):
|
||||
return int(line.strip().split('=')[1])
|
||||
except Exception as e:
|
||||
logger.error(f"Error reading DNS version: {e}")
|
||||
return 0
|
||||
|
||||
def get_status(self):
|
||||
params = {
|
||||
'token': TOKEN,
|
||||
'worker_config_version': self.config_version,
|
||||
'worker_dns_version': self.dns_version,
|
||||
'worker_version': WORKER_VERSION
|
||||
def interface_section(config):
|
||||
"""Return the [Interface] part of a WireGuard config, used to decide between syncconf and restart."""
|
||||
return config.split('[Peer]', 1)[0].strip()
|
||||
|
||||
|
||||
def read_file(path):
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as config_file:
|
||||
return config_file.read()
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
|
||||
|
||||
def write_file(path, content, mode=0o600, in_place=False):
|
||||
if in_place:
|
||||
with open(path, 'w', encoding='utf-8') as config_file:
|
||||
config_file.write(content)
|
||||
os.chmod(path, mode)
|
||||
return
|
||||
temp_path = f'{path}.tmp'
|
||||
with open(temp_path, 'w', encoding='utf-8') as config_file:
|
||||
config_file.write(content)
|
||||
os.chmod(temp_path, mode)
|
||||
os.replace(temp_path, path)
|
||||
|
||||
|
||||
def wireguard_status():
|
||||
result = run_command(['wg', 'show', 'all', 'dump'], check=False)
|
||||
if result.returncode != 0:
|
||||
return {}
|
||||
|
||||
data = {}
|
||||
# Peer lines: interface, public_key, preshared_key, endpoint, allowed_ips, latest_handshake, rx, tx, keepalive
|
||||
for line in result.stdout.strip().splitlines():
|
||||
parts = line.split('\t')
|
||||
if len(parts) < 9:
|
||||
continue
|
||||
interface, peer_public_key, endpoint, allowed_ips = parts[0], parts[1], parts[3], parts[4]
|
||||
data.setdefault(interface, {})[peer_public_key] = {
|
||||
'allowed-ips': [ip for ip in allowed_ips.split(',') if ip] if allowed_ips else [],
|
||||
'latest-handshakes': parts[5] or '',
|
||||
'transfer': {'tx': int(parts[7] or 0), 'rx': int(parts[6] or 0)},
|
||||
'endpoints': endpoint or '',
|
||||
}
|
||||
logger.info(f"Requesting status from Master... (Config Version: {self.config_version}, DNS Version: {self.dns_version}, Worker Version: {WORKER_VERSION})")
|
||||
try:
|
||||
response = self.session.get(f"{self.base_url}/status/", params=params, timeout=REQUEST_TIMEOUT)
|
||||
logger.info(f"Status response received. HTTP Code: {response.status_code}")
|
||||
return response
|
||||
except requests.RequestException as e:
|
||||
logger.error(f"Connection error while getting status: {e}")
|
||||
return None
|
||||
return data
|
||||
|
||||
def download_configs(self):
|
||||
params = {
|
||||
'token': TOKEN,
|
||||
'worker_config_version': self.config_version,
|
||||
'worker_dns_version': self.dns_version,
|
||||
'worker_version': WORKER_VERSION
|
||||
|
||||
def gateway_enabled():
|
||||
return os.path.isdir(GATEWAY_DIR) and os.path.isdir(CERTIFICATES_DIR)
|
||||
|
||||
|
||||
def http_service_is_running(url):
|
||||
try:
|
||||
requests.get(url, timeout=3)
|
||||
return True
|
||||
except requests.RequestException:
|
||||
return False
|
||||
|
||||
|
||||
def _parse_not_after(value):
|
||||
return datetime.strptime(value, '%b %d %H:%M:%S %Y %Z').replace(tzinfo=timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _fetch_certificate(hostname, verify):
|
||||
if verify:
|
||||
context = ssl.create_default_context()
|
||||
else:
|
||||
context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
|
||||
context.check_hostname = False
|
||||
context.verify_mode = ssl.CERT_NONE
|
||||
with socket.create_connection(('127.0.0.1', 443), timeout=3) as raw_socket:
|
||||
with context.wrap_socket(raw_socket, server_hostname=hostname) as tls_socket:
|
||||
return tls_socket.getpeercert() if verify else tls_socket.getpeercert(binary_form=True)
|
||||
|
||||
|
||||
def inspect_certificate(hostname):
|
||||
"""Same checks as app_gateway/certificate_status.py on the primary server, against the local Caddy."""
|
||||
try:
|
||||
certificate = _fetch_certificate(hostname, verify=True)
|
||||
issuer = dict(field[0] for field in certificate.get('issuer', ()))
|
||||
return {
|
||||
'status': 'trusted',
|
||||
'issuer': issuer.get('organizationName') or issuer.get('commonName') or '',
|
||||
'not_after': _parse_not_after(certificate['notAfter']),
|
||||
}
|
||||
try:
|
||||
response = self.session.get(f"{self.base_url}/worker/get_config_files/", params=params, timeout=REQUEST_TIMEOUT)
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
if data.get('status') == 'success':
|
||||
return data
|
||||
return None
|
||||
except requests.RequestException as e:
|
||||
logger.error(f"Error downloading configs: {e}")
|
||||
return None
|
||||
except ssl.SSLCertVerificationError:
|
||||
pass
|
||||
except (OSError, ValueError):
|
||||
return {'status': 'unavailable', 'issuer': '', 'not_after': None}
|
||||
|
||||
def download_dns_config(self):
|
||||
params = {
|
||||
'token': TOKEN,
|
||||
'worker_config_version': self.config_version,
|
||||
'worker_dns_version': self.dns_version,
|
||||
'worker_version': WORKER_VERSION
|
||||
}
|
||||
try:
|
||||
logger.info(f"Downloading DNS config... (Current Version: {self.dns_version})")
|
||||
response = self.session.get(f"{self.base_url}/worker/get_dnsmasq_config/", params=params, timeout=REQUEST_TIMEOUT)
|
||||
if response.status_code == 200:
|
||||
tar_path = 'dnsmasq_config.tar.gz'
|
||||
with open(tar_path, 'wb') as f:
|
||||
f.write(response.content)
|
||||
|
||||
logger.info("Extracting DNS config...")
|
||||
if not os.path.exists(DNS_DIR):
|
||||
os.makedirs(DNS_DIR, exist_ok=True)
|
||||
|
||||
subprocess.run(['tar', 'xvfz', tar_path, '-C', DNS_DIR], check=True, capture_output=True)
|
||||
if os.path.exists(tar_path):
|
||||
os.remove(tar_path)
|
||||
|
||||
# Update version
|
||||
self.dns_version = self.get_local_dns_version()
|
||||
logger.info(f"DNS config updated (New Version: {self.dns_version})")
|
||||
return True
|
||||
else:
|
||||
logger.error(f"Failed to download DNS config. Status: {response.status_code}")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating DNS config: {e}")
|
||||
return False
|
||||
try:
|
||||
certificate_der = _fetch_certificate(hostname, verify=False)
|
||||
result = subprocess.run(
|
||||
['openssl', 'x509', '-inform', 'DER', '-noout', '-issuer', '-enddate', '-nameopt', 'RFC2253'],
|
||||
input=certificate_der, capture_output=True, timeout=5,
|
||||
)
|
||||
except (OSError, ValueError, subprocess.SubprocessError):
|
||||
return {'status': 'unavailable', 'issuer': '', 'not_after': None}
|
||||
issuer = ''
|
||||
not_after = None
|
||||
for line in result.stdout.decode(errors='replace').splitlines():
|
||||
if line.startswith('issuer='):
|
||||
parts = dict(part.split('=', 1) for part in line[len('issuer='):].split(',') if '=' in part)
|
||||
issuer = INTERNAL_CA_NAME if INTERNAL_CA_NAME in line else parts.get('O') or parts.get('CN') or ''
|
||||
elif line.startswith('notAfter='):
|
||||
not_after = _parse_not_after(line[len('notAfter='):])
|
||||
return {'status': 'untrusted', 'issuer': issuer, 'not_after': not_after}
|
||||
|
||||
def send_ping(self):
|
||||
params = {
|
||||
'token': TOKEN,
|
||||
'worker_config_version': self.config_version,
|
||||
'worker_dns_version': self.dns_version,
|
||||
'worker_version': WORKER_VERSION
|
||||
}
|
||||
try:
|
||||
logger.info("Sending ping to Master...")
|
||||
self.session.get(f"{self.base_url}/worker/ping/", params=params, timeout=REQUEST_TIMEOUT)
|
||||
except Exception as e:
|
||||
logger.error(f"Error sending ping: {e}")
|
||||
|
||||
def apply_configs(self, data):
|
||||
logger.info("Applying new configurations...")
|
||||
files = data.get('files', {})
|
||||
cluster_settings = data.get('cluster_settings', {})
|
||||
new_config_version = cluster_settings.get('config_version', 0)
|
||||
|
||||
# 1. Stop existing interfaces
|
||||
self.cleanup_wireguard()
|
||||
|
||||
# 2. Write new files
|
||||
for filename, content in files.items():
|
||||
if filename == 'wg-firewall.sh' and isinstance(content, str):
|
||||
content = content.replace('wireguard-webadmin-dns', 'cluster-node-dns')
|
||||
filepath = os.path.join(WIREGUARD_DIR, filename)
|
||||
with open(filepath, 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
if filename == 'wg-firewall.sh':
|
||||
os.chmod(filepath, 0o755)
|
||||
|
||||
# Start interfaces
|
||||
conf_files = glob.glob(f"{WIREGUARD_DIR}/*.conf")
|
||||
for conf in conf_files:
|
||||
interface = os.path.basename(conf).replace('.conf', '')
|
||||
logger.info(f"Starting WireGuard interface: {interface}")
|
||||
def dns_is_running():
|
||||
# dnsmasq always listens on loopback when it is restricted to specific interfaces. The DNS container pauses
|
||||
# for a few seconds while reloading, so a missing answer is retried before reporting it as stopped.
|
||||
query = b'\x12\x34\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x09localhost\x00\x00\x01\x00\x01'
|
||||
for attempt in range(4):
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as dns_socket:
|
||||
dns_socket.settimeout(2)
|
||||
try:
|
||||
subprocess.run(['wg-quick', 'up', interface], check=True, capture_output=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.error(f"Failed to start {interface}: {e.stderr.decode()}")
|
||||
dns_socket.sendto(query, ('127.0.0.1', 53))
|
||||
dns_socket.recvfrom(512)
|
||||
return True
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(1)
|
||||
return False
|
||||
|
||||
# 4. Update config version
|
||||
self.config_version = new_config_version
|
||||
logger.info(f"Configuration updated to version {self.config_version}")
|
||||
|
||||
class ClusterNodeAgent:
|
||||
def __init__(self):
|
||||
self.base_url = f"{MASTER_SERVER_ADDRESS.rstrip('/')}/api/cluster/v2"
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update({
|
||||
'Authorization': f'Bearer {TOKEN}',
|
||||
'X-Worker-Version': str(WORKER_VERSION),
|
||||
})
|
||||
# Nothing is persisted: after a restart interfaces and firewall rules are gone, so every section is applied again
|
||||
self.applied_hashes = {}
|
||||
self.stats_interval = DEFAULT_STATS_INTERVAL
|
||||
self.wireguard_active = False
|
||||
|
||||
def request(self, method, path, **kwargs):
|
||||
try:
|
||||
return self.session.request(method, f'{self.base_url}/{path}', timeout=REQUEST_TIMEOUT, **kwargs)
|
||||
except requests.RequestException as error:
|
||||
logger.error('Request to %s failed: %s', path, error)
|
||||
return None
|
||||
|
||||
# Section handlers
|
||||
|
||||
def apply_firewall(self, section):
|
||||
content = section['files'].get('wg-firewall.sh')
|
||||
if content is None:
|
||||
raise SectionError('wg-firewall.sh missing from section')
|
||||
write_file(FIREWALL_SCRIPT, content, mode=0o700)
|
||||
run_command(['bash', FIREWALL_SCRIPT])
|
||||
|
||||
def apply_wireguard(self, section):
|
||||
desired_files = {
|
||||
filename: content for filename, content in section['files'].items()
|
||||
if filename.startswith('wg') and filename.endswith('.conf') and '/' not in filename
|
||||
}
|
||||
errors = []
|
||||
|
||||
for config_path in glob.glob(os.path.join(WIREGUARD_DIR, 'wg[0-9]*.conf')):
|
||||
filename = os.path.basename(config_path)
|
||||
if filename in desired_files:
|
||||
continue
|
||||
interface = filename[:-len('.conf')]
|
||||
if interface_is_up(interface):
|
||||
run_command(['wg-quick', 'down', interface], check=False)
|
||||
os.remove(config_path)
|
||||
logger.info('Removed WireGuard interface %s', interface)
|
||||
|
||||
for filename, content in sorted(desired_files.items()):
|
||||
interface = filename[:-len('.conf')]
|
||||
config_path = os.path.join(WIREGUARD_DIR, filename)
|
||||
previous_content = read_file(config_path)
|
||||
write_file(config_path, content)
|
||||
try:
|
||||
if not interface_is_up(interface):
|
||||
run_command(['wg-quick', 'up', interface])
|
||||
logger.info('Started WireGuard interface %s', interface)
|
||||
elif previous_content is None or interface_section(previous_content) != interface_section(content):
|
||||
run_command(['wg-quick', 'down', interface], check=False)
|
||||
run_command(['wg-quick', 'up', interface])
|
||||
logger.info('Restarted WireGuard interface %s', interface)
|
||||
elif previous_content != content:
|
||||
self.sync_peers(interface, config_path)
|
||||
logger.info('Synchronized peers of WireGuard interface %s', interface)
|
||||
except SectionError as error:
|
||||
errors.append(str(error))
|
||||
|
||||
self.wireguard_active = True
|
||||
if errors:
|
||||
raise SectionError('; '.join(errors))
|
||||
|
||||
@staticmethod
|
||||
def sync_peers(interface, config_path):
|
||||
stripped_config = run_command(['wg-quick', 'strip', config_path]).stdout
|
||||
with tempfile.NamedTemporaryFile('w', suffix='.conf', delete=False) as stripped_file:
|
||||
stripped_file.write(stripped_config)
|
||||
try:
|
||||
run_command(['wg', 'syncconf', interface, stripped_file.name])
|
||||
finally:
|
||||
os.remove(stripped_file.name)
|
||||
|
||||
def apply_dns(self, section):
|
||||
files = section['files']
|
||||
if DNS_MAIN_CONFIG not in files:
|
||||
raise SectionError(f'{DNS_MAIN_CONFIG} missing from section')
|
||||
os.makedirs(DNS_DIR, exist_ok=True)
|
||||
for filename, content in files.items():
|
||||
if filename != DNS_MAIN_CONFIG and '/' not in filename:
|
||||
write_file(os.path.join(DNS_DIR, filename), content, mode=0o644)
|
||||
for config_path in glob.glob(os.path.join(DNS_DIR, '*.conf')):
|
||||
if os.path.basename(config_path) not in files:
|
||||
os.remove(config_path)
|
||||
# Written last and in place: the DNS container watches this file with inotify to reload
|
||||
write_file(os.path.join(DNS_DIR, DNS_MAIN_CONFIG), files[DNS_MAIN_CONFIG], mode=0o644, in_place=True)
|
||||
|
||||
def apply_settings(self, section):
|
||||
self.stats_interval = int(section['values'].get('stats_sync_interval', DEFAULT_STATS_INTERVAL))
|
||||
|
||||
def apply_certificates(self, section):
|
||||
if not gateway_enabled():
|
||||
return
|
||||
files = {
|
||||
filename: content for filename, content in section['files'].items()
|
||||
if CERTIFICATE_FILE_PATTERN.match(filename)
|
||||
}
|
||||
for filename, content in files.items():
|
||||
certificate_path = os.path.join(CERTIFICATES_DIR, filename)
|
||||
if read_file(certificate_path) != content:
|
||||
write_file(certificate_path, content, mode=0o600)
|
||||
for certificate_path in glob.glob(os.path.join(CERTIFICATES_DIR, '*')):
|
||||
if os.path.basename(certificate_path) not in files:
|
||||
os.remove(certificate_path)
|
||||
|
||||
def apply_gateway(self, section):
|
||||
if not gateway_enabled():
|
||||
return
|
||||
for filename in GATEWAY_FILES:
|
||||
if filename not in section['files']:
|
||||
raise SectionError(f'{filename} missing from section')
|
||||
for filename in GATEWAY_FILES:
|
||||
write_file(os.path.join(GATEWAY_DIR, filename), section['files'][filename], mode=0o644)
|
||||
|
||||
SECTION_HANDLERS = {
|
||||
'firewall': apply_firewall,
|
||||
'wireguard': apply_wireguard,
|
||||
'dns': apply_dns,
|
||||
'settings': apply_settings,
|
||||
'certificates': apply_certificates,
|
||||
'gateway': apply_gateway,
|
||||
}
|
||||
|
||||
# Main loop
|
||||
|
||||
def shutdown_wireguard(self):
|
||||
if not self.wireguard_active:
|
||||
return
|
||||
logger.warning('Shutting down WireGuard interfaces and firewall rules')
|
||||
for config_path in glob.glob(os.path.join(WIREGUARD_DIR, 'wg[0-9]*.conf')):
|
||||
run_command(['wg-quick', 'down', os.path.basename(config_path)[:-len('.conf')]], check=False)
|
||||
for table, chain in FIREWALL_CHAINS:
|
||||
run_command(['iptables', '-t', table, '-F', chain], check=False)
|
||||
self.applied_hashes.pop('wireguard', None)
|
||||
self.applied_hashes.pop('firewall', None)
|
||||
self.wireguard_active = False
|
||||
|
||||
def sync_configuration(self):
|
||||
response = self.request('GET', 'manifest/')
|
||||
if response is None:
|
||||
return True
|
||||
if response.status_code == 403:
|
||||
logger.error('Token rejected by the primary server. Stopping.')
|
||||
self.shutdown_wireguard()
|
||||
return False
|
||||
if response.status_code == 400:
|
||||
logger.warning('Primary server refused this worker: %s', response.json().get('message', ''))
|
||||
self.shutdown_wireguard()
|
||||
return True
|
||||
if response.status_code != 200:
|
||||
logger.error('Unexpected manifest response: HTTP %s', response.status_code)
|
||||
return True
|
||||
|
||||
manifest = response.json().get('sections', {})
|
||||
reports = {}
|
||||
for section_name, section_hash in manifest.items():
|
||||
if section_name not in self.SECTION_HANDLERS or self.applied_hashes.get(section_name) == section_hash:
|
||||
continue
|
||||
section_response = self.request('GET', f'section/{section_name}/')
|
||||
if section_response is None or section_response.status_code != 200:
|
||||
continue
|
||||
section = section_response.json()
|
||||
try:
|
||||
self.SECTION_HANDLERS[section_name](self, section)
|
||||
self.applied_hashes[section_name] = section['hash']
|
||||
reports[section_name] = {'hash': section['hash'], 'status': 'ok'}
|
||||
logger.info('Section %s applied', section_name)
|
||||
except (SectionError, OSError, ValueError) as error:
|
||||
# Forget the last applied content: the next version must be applied and reported even if it
|
||||
# is identical to the one that was applied before the failure
|
||||
self.applied_hashes.pop(section_name, None)
|
||||
reports[section_name] = {'hash': section['hash'], 'status': 'error', 'message': str(error)}
|
||||
logger.error('Section %s failed: %s', section_name, error)
|
||||
|
||||
if reports:
|
||||
# Services are reported with the periodic stats: right after a DNS change dnsmasq is still reloading
|
||||
self.request('POST', 'report/', json={'sections': reports})
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def services_status():
|
||||
services = {'dnsmasq': 'running' if dns_is_running() else 'stopped'}
|
||||
if gateway_enabled():
|
||||
services['caddy'] = 'running' if http_service_is_running(CADDY_ADMIN_URL) else 'stopped'
|
||||
services['auth_gateway'] = 'running' if http_service_is_running(AUTH_GATEWAY_URL) else 'stopped'
|
||||
return services
|
||||
|
||||
@staticmethod
|
||||
def certificates_status():
|
||||
try:
|
||||
with open(os.path.join(GATEWAY_DIR, 'tls.json'), 'r', encoding='utf-8') as tls_file:
|
||||
hosts = json.load(tls_file).get('hosts', {})
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
return {hostname: inspect_certificate(hostname) for hostname in sorted(hosts)}
|
||||
|
||||
def send_stats(self):
|
||||
try:
|
||||
stats = self.func_process_wireguard_status()
|
||||
params = {
|
||||
'token': TOKEN,
|
||||
'worker_config_version': self.config_version,
|
||||
'worker_dns_version': self.dns_version,
|
||||
'worker_version': WORKER_VERSION
|
||||
}
|
||||
logger.info("Sending WireGuard stats to Master...")
|
||||
response = self.session.post(f"{self.base_url}/worker/submit_wireguard_stats/", json=stats, params=params, timeout=REQUEST_TIMEOUT)
|
||||
|
||||
if response.status_code == 200:
|
||||
logger.info("Stats sent successfully.")
|
||||
return True
|
||||
else:
|
||||
logger.error(f"Failed to send stats. Status: {response.status_code}")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Error sending stats: {e}")
|
||||
return False
|
||||
report = {
|
||||
'wireguard_status': wireguard_status(),
|
||||
'services': self.services_status(),
|
||||
}
|
||||
if gateway_enabled():
|
||||
report['certificates'] = self.certificates_status()
|
||||
self.request('POST', 'report/', json=report)
|
||||
|
||||
@staticmethod
|
||||
def publish_network_namespace():
|
||||
# Containers sharing this network namespace (DNS, Caddy) restart themselves when it changes
|
||||
network_namespace = os.readlink('/proc/self/ns/net')
|
||||
for directory in (DNS_DIR, GATEWAY_DIR):
|
||||
if os.path.isdir(directory):
|
||||
write_file(os.path.join(directory, '.netns_owner'), network_namespace + '\n', mode=0o644)
|
||||
|
||||
def run(self):
|
||||
if not self.should_run:
|
||||
return
|
||||
|
||||
logger.info("Cluster Worker starting...")
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_wireguard()
|
||||
|
||||
last_config_check = 0
|
||||
last_stats_send = 0
|
||||
stats_sync_interval = 60 # Default initial value
|
||||
logger.info('Cluster node agent %s starting', WORKER_VERSION)
|
||||
self.publish_network_namespace()
|
||||
self.wireguard_active = True
|
||||
self.shutdown_wireguard()
|
||||
|
||||
last_manifest = 0
|
||||
last_stats = 0
|
||||
while True:
|
||||
current_time = time.time()
|
||||
|
||||
# Check Config (Fixed 60s interval)
|
||||
if current_time - last_config_check >= 60:
|
||||
try:
|
||||
response = self.get_status()
|
||||
last_config_check = time.time()
|
||||
|
||||
if response is not None:
|
||||
if response.status_code == 403:
|
||||
logger.error("Received 403 Forbidden (Token invalid/deleted). Deactivating WireGuard and stopping requests permanently.")
|
||||
self.cleanup_wireguard()
|
||||
self.config_version = 0
|
||||
self.should_run = False
|
||||
break
|
||||
|
||||
if response.status_code == 400:
|
||||
logger.warning("Received 400 Bad Request (Worker suspended or error). Deactivating WireGuard and Firewall, but will keep retrying...")
|
||||
self.cleanup_wireguard()
|
||||
self.config_version = 0
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
|
||||
# Update stats interval from master settings
|
||||
cluster_settings = data.get('cluster_settings', {})
|
||||
stats_sync_interval = cluster_settings.get('stats_sync_interval', 60)
|
||||
|
||||
remote_config_version = cluster_settings.get('config_version', 0)
|
||||
|
||||
# Check WireGuard Config
|
||||
updated = False
|
||||
if int(remote_config_version) != self.config_version:
|
||||
logger.info(f"Config version mismatch (Local: {self.config_version}, Remote: {remote_config_version}). Updating...")
|
||||
config_data = self.download_configs()
|
||||
if config_data:
|
||||
self.apply_configs(config_data)
|
||||
self.send_ping()
|
||||
updated = True
|
||||
else:
|
||||
logger.error("Failed to download config files.")
|
||||
|
||||
# Check DNS Config
|
||||
remote_dns_version = int(cluster_settings.get('dns_version', 0))
|
||||
if not updated and remote_dns_version != self.dns_version:
|
||||
logger.info(f"DNS version mismatch (Local: {self.dns_version}, Remote: {remote_dns_version}). Updating...")
|
||||
if self.download_dns_config():
|
||||
self.send_ping()
|
||||
updated = True
|
||||
|
||||
if not updated and int(remote_config_version) == self.config_version and remote_dns_version == self.dns_version:
|
||||
logger.info(f"No changes detected. Configuration is up to date (WG: {self.config_version}, DNS: {self.dns_version}).")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error in config check loop: {e}")
|
||||
|
||||
# Check Stats Sync
|
||||
if current_time - last_stats_send >= stats_sync_interval:
|
||||
now = time.monotonic()
|
||||
if now - last_manifest >= MANIFEST_INTERVAL or not last_manifest:
|
||||
last_manifest = now
|
||||
if not self.sync_configuration():
|
||||
break
|
||||
if now - last_stats >= self.stats_interval:
|
||||
last_stats = now
|
||||
self.send_stats()
|
||||
last_stats_send = time.time()
|
||||
|
||||
# Sleep briefly to be responsive but not busy loop
|
||||
time.sleep(1)
|
||||
|
||||
# Final loop state if 403 was received
|
||||
while not self.should_run:
|
||||
logger.info("Worker disabled due to 403 Forbidden. WireGuard is off.")
|
||||
time.sleep(60)
|
||||
while True:
|
||||
time.sleep(3600)
|
||||
|
||||
if __name__ == "__main__":
|
||||
worker = ClusterWorker()
|
||||
worker.run()
|
||||
|
||||
if __name__ == '__main__':
|
||||
if not MASTER_SERVER_ADDRESS or not TOKEN:
|
||||
logger.error('MASTER_SERVER_ADDRESS and TOKEN must be set')
|
||||
else:
|
||||
ClusterNodeAgent().run()
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
if [[ "$COMPOSE_VERSION" != "c1b" ]]; then
|
||||
if [[ "$COMPOSE_VERSION" != "c2a" ]]; then
|
||||
echo "ERROR: Please upgrade your docker compose file. Exiting."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user