import glob import ipaddress import json import logging import os import re import socket import ssl import subprocess import tempfile import time from datetime import datetime, timezone import requests logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) MASTER_SERVER_ADDRESS = os.environ.get('MASTER_SERVER_ADDRESS', '') TOKEN = os.environ.get('TOKEN', '') WORKER_VERSION = 13 REQUEST_TIMEOUT = 30 MANIFEST_INTERVAL = 60 DEFAULT_STATS_INTERVAL = 60 WIREGUARD_DIR = '/etc/wireguard' FIREWALL_SCRIPT = os.path.join(WIREGUARD_DIR, 'wg-firewall.sh') DNS_DIR = '/etc/dnsmasq' DNS_MAIN_CONFIG = 'wireguard_webadmin_dns.conf' # Only mounted when the node runs the App Gateway (cluster_node-gateway.yml) GATEWAY_DIR = '/caddy_json_export' CERTIFICATES_DIR = '/caddy_certificates' GATEWAY_FILES = ('applications.json', 'auth_policies.json', 'routes.json', 'tls.json') CERTIFICATE_FILE_PATTERN = re.compile(r'^[a-f0-9]{64}\.(crt|key)$') CADDY_ADMIN_URL = 'http://127.0.0.1:2019/config/' AUTH_GATEWAY_URL = 'http://cluster-node-auth-gateway:9091/' INTERNAL_CA_NAME = 'Caddy Local Authority' FIREWALL_CHAINS = ( ('nat', 'WGWADM_POSTROUTING'), ('nat', 'WGWADM_PREROUTING'), ('filter', 'WGWADM_FORWARD'), ('filter', 'WGWADM_ROUTE_POLICY'), ) class SectionError(Exception): pass def run_command(command, check=True): result = subprocess.run(command, capture_output=True, text=True) if check and result.returncode != 0: raise SectionError(f"{' '.join(command)}: {(result.stderr or result.stdout).strip()}") return result def interface_is_up(interface): return run_command(['wg', 'show', interface], check=False).returncode == 0 def sync_interface_routes(interface): """ Bring the IPv4 routes of a running interface in line with its allowed IPs, as wg-quick does on start. wg syncconf only updates the peers. Default routes (/0) are left to wg-quick. """ allowed_networks = set() for line in run_command(['wg', 'show', interface, 'allowed-ips']).stdout.splitlines(): for value in line.split()[1:]: try: network = ipaddress.IPv4Network(value, strict=False) except ValueError: continue if network.prefixlen: allowed_networks.add(network) allowed_networks = sorted(allowed_networks, key=lambda network: network.prefixlen) errors = [] for network in allowed_networks: if run_command(['ip', '-4', 'route', 'show', 'dev', interface, 'match', str(network)], check=False).stdout.strip(): continue # replace instead of add: the network may still be routed to another instance it was moved from result = run_command(['ip', '-4', 'route', 'replace', str(network), 'dev', interface], check=False) if result.returncode != 0: errors.append(f'{network}: {result.stderr.strip()}') for route in run_command(['ip', '-o', '-4', 'route', 'show', 'dev', interface], check=False).stdout.splitlines(): fields = route.split() if not fields or 'kernel' in fields: continue try: route_network = ipaddress.IPv4Network(fields[0], strict=False) except ValueError: continue # Only routes that no allowed IP covers are removed: the interface drops their traffic anyway if any(route_network.subnet_of(network) for network in allowed_networks): continue result = run_command(['ip', '-4', 'route', 'delete', str(route_network), 'dev', interface], check=False) if result.returncode != 0: errors.append(f'{route_network}: {result.stderr.strip()}') if errors: raise SectionError(f"{interface} routes: {'; '.join(errors)}") def interface_section(config): """Return the [Interface] part of a WireGuard config, used to decide between syncconf and restart.""" return config.split('[Peer]', 1)[0].strip() def read_file(path): try: with open(path, 'r', encoding='utf-8') as config_file: return config_file.read() except FileNotFoundError: return None def write_file(path, content, mode=0o600, in_place=False): if in_place: with open(path, 'w', encoding='utf-8') as config_file: config_file.write(content) os.chmod(path, mode) return temp_path = f'{path}.tmp' with open(temp_path, 'w', encoding='utf-8') as config_file: config_file.write(content) os.chmod(temp_path, mode) os.replace(temp_path, path) def wireguard_status(): result = run_command(['wg', 'show', 'all', 'dump'], check=False) if result.returncode != 0: return {} data = {} # Peer lines: interface, public_key, preshared_key, endpoint, allowed_ips, latest_handshake, rx, tx, keepalive for line in result.stdout.strip().splitlines(): parts = line.split('\t') if len(parts) < 9: continue interface, peer_public_key, endpoint, allowed_ips = parts[0], parts[1], parts[3], parts[4] data.setdefault(interface, {})[peer_public_key] = { 'allowed-ips': [ip for ip in allowed_ips.split(',') if ip] if allowed_ips else [], 'latest-handshakes': parts[5] or '', 'transfer': {'tx': int(parts[7] or 0), 'rx': int(parts[6] or 0)}, 'endpoints': endpoint or '', } return data def gateway_enabled(): return os.path.isdir(GATEWAY_DIR) and os.path.isdir(CERTIFICATES_DIR) def http_service_is_running(url): try: requests.get(url, timeout=3) return True except requests.RequestException: return False def _parse_not_after(value): return datetime.strptime(value, '%b %d %H:%M:%S %Y %Z').replace(tzinfo=timezone.utc).isoformat() def _fetch_certificate(hostname, verify): if verify: context = ssl.create_default_context() else: context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) context.check_hostname = False context.verify_mode = ssl.CERT_NONE with socket.create_connection(('127.0.0.1', 443), timeout=3) as raw_socket: with context.wrap_socket(raw_socket, server_hostname=hostname) as tls_socket: return tls_socket.getpeercert() if verify else tls_socket.getpeercert(binary_form=True) def inspect_certificate(hostname): """Same checks as app_gateway/certificate_status.py on the primary server, against the local Caddy.""" try: certificate = _fetch_certificate(hostname, verify=True) issuer = dict(field[0] for field in certificate.get('issuer', ())) return { 'status': 'trusted', 'issuer': issuer.get('organizationName') or issuer.get('commonName') or '', 'not_after': _parse_not_after(certificate['notAfter']), } except ssl.SSLCertVerificationError: pass except (OSError, ValueError): return {'status': 'unavailable', 'issuer': '', 'not_after': None} try: certificate_der = _fetch_certificate(hostname, verify=False) result = subprocess.run( ['openssl', 'x509', '-inform', 'DER', '-noout', '-issuer', '-enddate', '-nameopt', 'RFC2253'], input=certificate_der, capture_output=True, timeout=5, ) except (OSError, ValueError, subprocess.SubprocessError): return {'status': 'unavailable', 'issuer': '', 'not_after': None} issuer = '' not_after = None for line in result.stdout.decode(errors='replace').splitlines(): if line.startswith('issuer='): parts = dict(part.split('=', 1) for part in line[len('issuer='):].split(',') if '=' in part) issuer = INTERNAL_CA_NAME if INTERNAL_CA_NAME in line else parts.get('O') or parts.get('CN') or '' elif line.startswith('notAfter='): not_after = _parse_not_after(line[len('notAfter='):]) return {'status': 'untrusted', 'issuer': issuer, 'not_after': not_after} def dns_is_running(): # dnsmasq always listens on loopback when it is restricted to specific interfaces. The DNS container pauses # for a few seconds while reloading, so a missing answer is retried before reporting it as stopped. query = b'\x12\x34\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x09localhost\x00\x00\x01\x00\x01' for attempt in range(4): with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as dns_socket: dns_socket.settimeout(2) try: dns_socket.sendto(query, ('127.0.0.1', 53)) dns_socket.recvfrom(512) return True except OSError: pass time.sleep(1) return False class ClusterNodeAgent: def __init__(self): self.base_url = f"{MASTER_SERVER_ADDRESS.rstrip('/')}/api/cluster/v2" self.session = requests.Session() self.session.headers.update({ 'Authorization': f'Bearer {TOKEN}', 'X-Worker-Version': str(WORKER_VERSION), }) # Nothing is persisted: after a restart interfaces and firewall rules are gone, so every section is applied again self.applied_hashes = {} self.stats_interval = DEFAULT_STATS_INTERVAL self.wireguard_active = False def request(self, method, path, **kwargs): try: return self.session.request(method, f'{self.base_url}/{path}', timeout=REQUEST_TIMEOUT, **kwargs) except requests.RequestException as error: logger.error('Request to %s failed: %s', path, error) return None # Section handlers def apply_firewall(self, section): content = section['files'].get('wg-firewall.sh') if content is None: raise SectionError('wg-firewall.sh missing from section') write_file(FIREWALL_SCRIPT, content, mode=0o700) run_command(['bash', FIREWALL_SCRIPT]) def apply_wireguard(self, section): desired_files = { filename: content for filename, content in section['files'].items() if filename.startswith('wg') and filename.endswith('.conf') and '/' not in filename } errors = [] for config_path in glob.glob(os.path.join(WIREGUARD_DIR, 'wg[0-9]*.conf')): filename = os.path.basename(config_path) if filename in desired_files: continue interface = filename[:-len('.conf')] if interface_is_up(interface): run_command(['wg-quick', 'down', interface], check=False) os.remove(config_path) logger.info('Removed WireGuard interface %s', interface) for filename, content in sorted(desired_files.items()): interface = filename[:-len('.conf')] config_path = os.path.join(WIREGUARD_DIR, filename) previous_content = read_file(config_path) write_file(config_path, content) try: if not interface_is_up(interface): run_command(['wg-quick', 'up', interface]) logger.info('Started WireGuard interface %s', interface) elif previous_content is None or interface_section(previous_content) != interface_section(content): run_command(['wg-quick', 'down', interface], check=False) run_command(['wg-quick', 'up', interface]) logger.info('Restarted WireGuard interface %s', interface) elif previous_content != content: self.sync_peers(interface, config_path) logger.info('Synchronized peers of WireGuard interface %s', interface) except SectionError as error: errors.append(str(error)) # Back to the content actually applied, so the next attempt sees the change again if previous_content is not None: write_file(config_path, previous_content) self.wireguard_active = True if errors: raise SectionError('; '.join(errors)) @staticmethod def sync_peers(interface, config_path): stripped_config = run_command(['wg-quick', 'strip', config_path]).stdout with tempfile.NamedTemporaryFile('w', suffix='.conf', delete=False) as stripped_file: stripped_file.write(stripped_config) try: run_command(['wg', 'syncconf', interface, stripped_file.name]) finally: os.remove(stripped_file.name) sync_interface_routes(interface) def apply_dns(self, section): files = section['files'] if DNS_MAIN_CONFIG not in files: raise SectionError(f'{DNS_MAIN_CONFIG} missing from section') os.makedirs(DNS_DIR, exist_ok=True) for filename, content in files.items(): if filename != DNS_MAIN_CONFIG and '/' not in filename: write_file(os.path.join(DNS_DIR, filename), content, mode=0o644) for config_path in glob.glob(os.path.join(DNS_DIR, '*.conf')): if os.path.basename(config_path) not in files: os.remove(config_path) # Written last and in place: the DNS container watches this file with inotify to reload write_file(os.path.join(DNS_DIR, DNS_MAIN_CONFIG), files[DNS_MAIN_CONFIG], mode=0o644, in_place=True) def apply_settings(self, section): self.stats_interval = int(section['values'].get('stats_sync_interval', DEFAULT_STATS_INTERVAL)) def apply_certificates(self, section): if not gateway_enabled(): return files = { filename: content for filename, content in section['files'].items() if CERTIFICATE_FILE_PATTERN.match(filename) } for filename, content in files.items(): certificate_path = os.path.join(CERTIFICATES_DIR, filename) if read_file(certificate_path) != content: write_file(certificate_path, content, mode=0o600) for certificate_path in glob.glob(os.path.join(CERTIFICATES_DIR, '*')): if os.path.basename(certificate_path) not in files: os.remove(certificate_path) def apply_gateway(self, section): if not gateway_enabled(): return for filename in GATEWAY_FILES: if filename not in section['files']: raise SectionError(f'{filename} missing from section') for filename in GATEWAY_FILES: write_file(os.path.join(GATEWAY_DIR, filename), section['files'][filename], mode=0o644) SECTION_HANDLERS = { 'firewall': apply_firewall, 'wireguard': apply_wireguard, 'dns': apply_dns, 'settings': apply_settings, 'certificates': apply_certificates, 'gateway': apply_gateway, } # Main loop def shutdown_wireguard(self): if not self.wireguard_active: return logger.warning('Shutting down WireGuard interfaces and firewall rules') for config_path in glob.glob(os.path.join(WIREGUARD_DIR, 'wg[0-9]*.conf')): run_command(['wg-quick', 'down', os.path.basename(config_path)[:-len('.conf')]], check=False) for table, chain in FIREWALL_CHAINS: run_command(['iptables', '-t', table, '-F', chain], check=False) self.applied_hashes.pop('wireguard', None) self.applied_hashes.pop('firewall', None) self.wireguard_active = False def sync_configuration(self): response = self.request('GET', 'manifest/') if response is None: return True if response.status_code == 403: logger.error('Token rejected by the primary server. Stopping.') self.shutdown_wireguard() return False if response.status_code == 400: logger.warning('Primary server refused this worker: %s', response.json().get('message', '')) self.shutdown_wireguard() return True if response.status_code != 200: logger.error('Unexpected manifest response: HTTP %s', response.status_code) return True manifest = response.json().get('sections', {}) reports = {} for section_name, section_hash in manifest.items(): if section_name not in self.SECTION_HANDLERS or self.applied_hashes.get(section_name) == section_hash: continue section_response = self.request('GET', f'section/{section_name}/') if section_response is None or section_response.status_code != 200: continue section = section_response.json() try: self.SECTION_HANDLERS[section_name](self, section) self.applied_hashes[section_name] = section['hash'] reports[section_name] = {'hash': section['hash'], 'status': 'ok'} logger.info('Section %s applied', section_name) except (SectionError, OSError, ValueError) as error: # Forget the last applied content: the next version must be applied and reported even if it # is identical to the one that was applied before the failure self.applied_hashes.pop(section_name, None) reports[section_name] = {'hash': section['hash'], 'status': 'error', 'message': str(error)} logger.error('Section %s failed: %s', section_name, error) if reports: # Services are reported with the periodic stats: right after a DNS change dnsmasq is still reloading self.request('POST', 'report/', json={'sections': reports}) return True @staticmethod def services_status(): services = {'dnsmasq': 'running' if dns_is_running() else 'stopped'} if gateway_enabled(): services['caddy'] = 'running' if http_service_is_running(CADDY_ADMIN_URL) else 'stopped' services['auth_gateway'] = 'running' if http_service_is_running(AUTH_GATEWAY_URL) else 'stopped' return services @staticmethod def certificates_status(): try: with open(os.path.join(GATEWAY_DIR, 'tls.json'), 'r', encoding='utf-8') as tls_file: hosts = json.load(tls_file).get('hosts', {}) except (OSError, ValueError): return {} return {hostname: inspect_certificate(hostname) for hostname in sorted(hosts)} def send_stats(self): report = { 'wireguard_status': wireguard_status(), 'services': self.services_status(), } if gateway_enabled(): report['certificates'] = self.certificates_status() self.request('POST', 'report/', json=report) @staticmethod def publish_network_namespace(): # Containers sharing this network namespace (DNS, Caddy) restart themselves when it changes network_namespace = os.readlink('/proc/self/ns/net') for directory in (DNS_DIR, GATEWAY_DIR): if os.path.isdir(directory): write_file(os.path.join(directory, '.netns_owner'), network_namespace + '\n', mode=0o644) def run(self): logger.info('Cluster node agent %s starting', WORKER_VERSION) self.publish_network_namespace() self.wireguard_active = True self.shutdown_wireguard() last_manifest = 0 last_stats = 0 while True: now = time.monotonic() if now - last_manifest >= MANIFEST_INTERVAL or not last_manifest: last_manifest = now if not self.sync_configuration(): break if now - last_stats >= self.stats_interval: last_stats = now self.send_stats() time.sleep(1) while True: time.sleep(3600) if __name__ == '__main__': if not MASTER_SERVER_ADDRESS or not TOKEN: logger.error('MASTER_SERVER_ADDRESS and TOKEN must be set') else: ClusterNodeAgent().run()