Add distributed cluster mode and sync peer network routes on reload

This commit is contained in:
Eduardo Silva
2026-11-20 20:42:42 -03:00
parent 5f6e2cd61c
commit 0ee6c2b864
25 changed files with 993 additions and 70 deletions
+106
View File
@@ -1,11 +1,110 @@
import ipaddress
import os
import subprocess
from typing import Union
from django.utils.translation import gettext_lazy as _
from cluster.models import ClusterSettings, Worker
from wireguard.models import WireGuardInstance
WIREGUARD_CONFIG_DIR = '/etc/wireguard'
# Configurations of instances that run on workers (distributed cluster mode). Kept out of WIREGUARD_CONFIG_DIR
# so the primary server does not bring them up.
CLUSTER_WIREGUARD_CONFIG_DIR = '/etc/wireguard/cluster'
def get_instance_config_dir(instance: WireGuardInstance) -> str:
return CLUSTER_WIREGUARD_CONFIG_DIR if instance.hosting_worker else WIREGUARD_CONFIG_DIR
def get_config_servers(instance: WireGuardInstance) -> list:
"""Servers offered when downloading a peer configuration of the instance. Empty without the cluster."""
cluster_settings = ClusterSettings.objects.filter(name='cluster_settings', enabled=True).first()
if not cluster_settings:
return []
if ClusterSettings.is_distributed():
hosting_worker = instance.hosting_worker if instance else None
if hosting_worker:
return [{'name': hosting_worker.display_name, 'uuid': str(hosting_worker.uuid)}]
return [{'name': _('Primary Server'), 'uuid': ''}]
servers = []
if cluster_settings.primary_enable_wireguard:
servers.append({'name': _('Primary Server'), 'uuid': ''})
for worker in Worker.objects.filter(enabled=True):
servers.append({'name': worker.display_name, 'uuid': str(worker.uuid)})
return servers
def sync_interface_routes(interface_name: str) -> list:
"""
Bring the IPv4 routes of a running interface in line with its allowed IPs, as wg-quick does on start.
wg syncconf only updates the peers, so networks added to or removed from a peer would otherwise only
get their routes on the next restart. Default routes (/0) are left to wg-quick. Returns the errors found.
"""
errors = []
allowed_ips_result = subprocess.run(['wg', 'show', interface_name, 'allowed-ips'], capture_output=True, text=True)
if allowed_ips_result.returncode != 0:
return [allowed_ips_result.stderr.strip()]
allowed_networks = set()
for line in allowed_ips_result.stdout.splitlines():
for value in line.split()[1:]:
try:
network = ipaddress.IPv4Network(value, strict=False)
except ValueError:
continue
if network.prefixlen:
allowed_networks.add(network)
allowed_networks = sorted(allowed_networks, key=lambda network: network.prefixlen)
for network in allowed_networks:
covering_routes = subprocess.run(
['ip', '-4', 'route', 'show', 'dev', interface_name, 'match', str(network)], capture_output=True, text=True,
).stdout.strip()
if covering_routes:
continue
# replace instead of add: the network may still be routed to another instance it was moved from
add_result = subprocess.run(
['ip', '-4', 'route', 'replace', str(network), 'dev', interface_name], capture_output=True, text=True
)
if add_result.returncode != 0:
errors.append(f"{network}: {add_result.stderr.strip()}")
routes_result = subprocess.run(
['ip', '-o', '-4', 'route', 'show', 'dev', interface_name], capture_output=True, text=True
)
for route in routes_result.stdout.splitlines():
fields = route.split()
if not fields or 'kernel' in fields:
continue
try:
route_network = ipaddress.IPv4Network(fields[0], strict=False)
except ValueError:
continue
# Only routes that no allowed IP covers are removed: the interface drops their traffic anyway
if any(route_network.subnet_of(network) for network in allowed_networks):
continue
delete_result = subprocess.run(
['ip', '-4', 'route', 'delete', str(route_network), 'dev', interface_name], capture_output=True, text=True,
)
if delete_result.returncode != 0:
errors.append(f"{route_network}: {delete_result.stderr.strip()}")
return errors
def func_reload_wireguard_interface(target: Union[str, WireGuardInstance], config_dir: str = "/etc/wireguard") -> tuple[bool, str]:
success, message = reload_wireguard_interface(target, config_dir)
if not success and isinstance(target, WireGuardInstance):
# Keep the apply banner so the configuration is applied again with a restart
target.pending_changes = True
WireGuardInstance.objects.filter(pk=target.pk).update(pending_changes=True)
return success, message
def reload_wireguard_interface(target: Union[str, WireGuardInstance], config_dir: str) -> tuple[bool, str]:
"""
Reload a WireGuard interface safely using:
wg syncconf <iface> <(wg-quick strip <conf>)
@@ -21,6 +120,9 @@ def func_reload_wireguard_interface(target: Union[str, WireGuardInstance], confi
# Resolve interface name and config path
if isinstance(target, WireGuardInstance):
if target.hosting_worker:
# The worker applies the new configuration on its next synchronization
return True, f"wg{target.instance_id} runs on {target.hosting_worker} and is updated by the cluster"
interface_name = f"wg{target.instance_id}"
conf_filename = f"{interface_name}.conf"
else:
@@ -59,6 +161,10 @@ def func_reload_wireguard_interface(target: Union[str, WireGuardInstance], confi
if sync_proc.returncode != 0:
return False, sync_proc.stderr.strip()
route_errors = sync_interface_routes(interface_name)
if route_errors:
return False, f"{interface_name} reloaded with route errors: {'; '.join(route_errors)}"
return True, f"{interface_name} reloaded successfully"
except subprocess.CalledProcessError as e:
+256 -1
View File
@@ -1,3 +1,258 @@
import importlib.util
import ipaddress
import os
import subprocess
import tempfile
from unittest.mock import patch
from django.conf import settings
from django.contrib.auth.models import User
from django.contrib.messages import get_messages
from django.test import TestCase
# Create your tests here.
from user_manager.models import UserAcl
from wireguard.models import WireGuardInstance
from wireguard_tools import functions
def load_cluster_worker():
module_path = os.path.join(settings.BASE_DIR, 'containers', 'cluster_node', 'cluster_worker.py')
spec = importlib.util.spec_from_file_location('cluster_worker_under_test', module_path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
class FakeNetwork:
"""
Minimal stand-in for the wg and ip commands used to synchronize routes. Keeps the routes of one interface and
answers 'route show ... match' like iproute2: routes whose prefix contains the given network.
"""
def __init__(self, interface, allowed_ips, routes):
self.interface = interface
self.allowed_ips = allowed_ips
# {network: is_kernel_route}
self.routes = {ipaddress.ip_network(route): kernel for route, kernel in routes.items()}
self.commands = []
def route_lines(self):
return [
f'{network} proto kernel scope link src x' if kernel else f'{network} scope link '
for network, kernel in sorted(self.routes.items(), key=lambda item: str(item[0]))
]
def run(self, command, *args, **kwargs):
self.commands.append(command)
stdout = ''
if command[:2] == ['wg', 'show']:
stdout = ''.join(f'peer{index}\t{value}\n' for index, value in enumerate(self.allowed_ips))
elif command[0] == 'ip':
arguments = [value for value in command[1:] if value not in ('-o', '-4')]
action = arguments[1]
if action == 'show' and 'match' in arguments:
network = ipaddress.ip_network(arguments[-1])
stdout = '\n'.join(
line for line in self.route_lines()
if network.subnet_of(ipaddress.ip_network(line.split()[0]))
)
elif action == 'show':
stdout = '\n'.join(self.route_lines())
elif action == 'replace':
self.routes[ipaddress.ip_network(arguments[2])] = False
elif action == 'delete':
self.routes.pop(ipaddress.ip_network(arguments[2]), None)
return subprocess.CompletedProcess(command, 0, stdout, '')
def route_names(self):
return sorted(str(network) for network in self.routes)
class RouteSyncCases:
"""Cases shared by the primary server and the cluster node agent implementations."""
def sync(self, network):
raise NotImplementedError
def test_adds_route_of_new_allowed_network(self):
network = FakeNetwork('wg0', ['10.188.0.2/32 192.168.60.0/24'], {'10.188.0.0/24': True})
self.sync(network)
self.assertEqual(network.route_names(), ['10.188.0.0/24', '192.168.60.0/24'])
def test_does_not_add_route_already_covered(self):
network = FakeNetwork(
'wg0', ['10.188.0.2/32 192.168.15.128/25', '10.188.0.3/32 192.168.15.0/24'], {'10.188.0.0/24': True}
)
self.sync(network)
# Peer addresses are covered by the instance route and the /25 by the /24 added first
self.assertEqual(network.route_names(), ['10.188.0.0/24', '192.168.15.0/24'])
replaced = [command[4] for command in network.commands if command[0] == 'ip' and 'replace' in command]
self.assertEqual(replaced, ['192.168.15.0/24'])
def test_removes_route_not_covered_by_allowed_ips(self):
network = FakeNetwork(
'wg0', ['10.188.0.2/32 192.168.15.0/24'],
{'10.188.0.0/24': True, '192.168.15.0/24': False, '192.168.15.128/25': False, '192.168.99.0/24': False},
)
self.sync(network)
# The kernel route and the route covered by an allowed network stay
self.assertEqual(network.route_names(), ['10.188.0.0/24', '192.168.15.0/24', '192.168.15.128/25'])
def test_keeps_kernel_route_without_peers(self):
network = FakeNetwork('wg0', ['(none)'], {'10.188.0.0/24': True, '192.168.60.0/24': False})
self.sync(network)
self.assertEqual(network.route_names(), ['10.188.0.0/24'])
def test_ignores_default_route(self):
network = FakeNetwork('wg0', ['10.188.0.2/32 0.0.0.0/0'], {'10.188.0.0/24': True})
self.sync(network)
self.assertEqual(network.route_names(), ['10.188.0.0/24'])
self.assertFalse([command for command in network.commands if 'replace' in command])
class PrimaryRouteSyncTests(RouteSyncCases, TestCase):
def sync(self, network):
with patch('wireguard_tools.functions.subprocess.run', side_effect=network.run):
self.assertEqual(functions.sync_interface_routes(network.interface), [])
def test_reload_syncs_routes_after_syncconf(self):
network = FakeNetwork('wg0', ['10.188.0.2/32 192.168.60.0/24'], {'10.188.0.0/24': True})
with patch('wireguard_tools.functions.os.path.exists', return_value=True), \
patch('wireguard_tools.functions.os.remove'), \
patch('builtins.open'), \
patch('wireguard_tools.functions.subprocess.run', side_effect=network.run):
success, message = functions.func_reload_wireguard_interface('wg0')
self.assertTrue(success, message)
self.assertIn('192.168.60.0/24', network.route_names())
class AgentRouteSyncTests(RouteSyncCases, TestCase):
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.cluster_worker = load_cluster_worker()
def sync(self, network):
with patch.object(self.cluster_worker.subprocess, 'run', side_effect=network.run):
self.cluster_worker.sync_interface_routes(network.interface)
def test_sync_peers_syncs_routes(self):
network = FakeNetwork('wg0', ['10.188.0.2/32 192.168.60.0/24'], {'10.188.0.0/24': True})
with patch.object(self.cluster_worker.subprocess, 'run', side_effect=network.run):
self.cluster_worker.ClusterNodeAgent.sync_peers('wg0', '/etc/wireguard/wg0.conf')
self.assertIn('192.168.60.0/24', network.route_names())
class RestartInterfacesTests(TestCase):
def setUp(self):
user = User.objects.create_user(username='admin', password='unused')
UserAcl.objects.create(user=user, user_level=50)
self.client.force_login(user)
def test_restart_does_not_stop_interface_that_is_not_running(self):
commands = []
def run(command, *args, **kwargs):
commands.append(command if isinstance(command, str) else ' '.join(command))
if command == ['wg', 'show', 'interfaces']:
return subprocess.CompletedProcess(command, 0, 'wg0\n', '')
return subprocess.CompletedProcess(command, 0, '', '')
with patch('wireguard_tools.views.os.listdir', return_value=['wg0.conf', 'wg1.conf']), \
patch('wireguard_tools.views.os.path.isfile', return_value=True), \
patch('wireguard_tools.views.subprocess.run', side_effect=run):
response = self.client.get('/tools/restart_wireguard/')
self.assertIn('wg-quick down wg0', commands)
self.assertNotIn('wg-quick down wg1', commands)
self.assertIn('wg-quick up wg1', commands)
message_texts = [str(message) for message in get_messages(response.wsgi_request)]
self.assertFalse([text for text in message_texts if 'Error' in text], message_texts)
class ReloadFailureTests(TestCase):
def setUp(self):
self.instance = WireGuardInstance.objects.create(
instance_id=0, private_key='private-0', public_key='public-0', hostname='primary.example.test',
listen_port=51820, address='10.188.0.1', pending_changes=False,
)
def test_failed_reload_marks_instance_pending(self):
with patch('wireguard_tools.functions.reload_wireguard_interface', return_value=(False, 'syncconf failed')):
success, message = functions.func_reload_wireguard_interface(self.instance)
self.assertFalse(success)
self.assertTrue(self.instance.pending_changes)
self.instance.refresh_from_db()
self.assertTrue(self.instance.pending_changes)
def test_successful_reload_keeps_instance_applied(self):
with patch('wireguard_tools.functions.reload_wireguard_interface', return_value=(True, 'wg0 reloaded')):
success, message = functions.func_reload_wireguard_interface(self.instance)
self.assertTrue(success)
self.instance.refresh_from_db()
self.assertFalse(self.instance.pending_changes)
def test_route_error_fails_the_reload(self):
network = FakeNetwork('wg0', ['10.188.0.2/32 192.168.60.0/24'], {'10.188.0.0/24': True})
def run(command, *args, **kwargs):
if 'replace' in command:
return subprocess.CompletedProcess(command, 2, '', 'RTNETLINK answers: failure')
return network.run(command)
with patch('wireguard_tools.functions.os.path.exists', return_value=True), \
patch('wireguard_tools.functions.os.remove'), \
patch('builtins.open'), \
patch('wireguard_tools.functions.subprocess.run', side_effect=run):
success, message = functions.func_reload_wireguard_interface(self.instance)
self.assertFalse(success)
self.assertIn('192.168.60.0/24', message)
self.instance.refresh_from_db()
self.assertTrue(self.instance.pending_changes)
class AgentInterfaceFailureTests(TestCase):
"""A failed interface must be applied again on the next attempt, even if the content did not change."""
OLD_CONFIG = '[Interface]\nListenPort = 51820\n\n[Peer]\nPublicKey = peer-a\nAllowedIPs = 10.188.0.2/32\n'
NEW_CONFIG = OLD_CONFIG + 'AllowedIPs = 192.168.60.0/24\n'
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.cluster_worker = load_cluster_worker()
def setUp(self):
temp_dir = tempfile.TemporaryDirectory()
self.addCleanup(temp_dir.cleanup)
self.config_path = os.path.join(temp_dir.name, 'wg0.conf')
with open(self.config_path, 'w', encoding='utf-8') as config_file:
config_file.write(self.OLD_CONFIG)
for name, value in (('WIREGUARD_DIR', temp_dir.name), ('interface_is_up', lambda interface: True)):
patcher = patch.object(self.cluster_worker, name, value)
patcher.start()
self.addCleanup(patcher.stop)
self.agent = self.cluster_worker.ClusterNodeAgent()
self.section = {'files': {'wg0.conf': self.NEW_CONFIG}}
def read_config(self):
with open(self.config_path, encoding='utf-8') as config_file:
return config_file.read()
def test_failed_sync_restores_previous_content_and_retries(self):
SectionError = self.cluster_worker.SectionError
with patch.object(self.cluster_worker.ClusterNodeAgent, 'sync_peers', side_effect=SectionError('wg0 routes: failure')):
with self.assertRaises(SectionError):
self.agent.apply_wireguard(self.section)
self.assertEqual(self.read_config(), self.OLD_CONFIG)
with patch.object(self.cluster_worker.ClusterNodeAgent, 'sync_peers') as sync_peers:
self.agent.apply_wireguard(self.section)
sync_peers.assert_called_once_with('wg0', self.config_path)
self.assertEqual(self.read_config(), self.NEW_CONFIG)
def test_unchanged_content_is_not_applied_again(self):
with patch.object(self.cluster_worker.ClusterNodeAgent, 'sync_peers') as sync_peers:
self.agent.apply_wireguard(self.section)
self.agent.apply_wireguard(self.section)
sync_peers.assert_called_once()
+66 -33
View File
@@ -15,13 +15,16 @@ from django.shortcuts import get_object_or_404, redirect, render
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from cluster.models import Worker
from cluster.models import ClusterSettings, Worker
from firewall.models import RedirectRule, FirewallSettings
from firewall.tools import generate_firewall_script
from user_manager.models import UserAcl
from vpn_invite.models import PeerInvite
from wgwadmlibrary.tools import user_has_access_to_peer
from wireguard.models import Peer, PeerAllowedIP, WireGuardInstance
from wireguard_tools.functions import (
CLUSTER_WIREGUARD_CONFIG_DIR, WIREGUARD_CONFIG_DIR, get_instance_config_dir, sync_interface_routes
)
def clean_command_field(command_field):
@@ -45,7 +48,11 @@ def generate_peer_config(peer_uuid, server_address=None):
if dns_line and wg_instance.dns_search_domains:
dns_line += f", {wg_instance.dns_search_domains}"
if server_address:
hosting_worker = wg_instance.hosting_worker
if hosting_worker and hosting_worker.server_address:
# Distributed cluster mode: the instance only runs on this worker
endpoint = f"{hosting_worker.server_address}:{wg_instance.listen_port}"
elif server_address:
endpoint = server_address
else:
endpoint = f"{wg_instance.hostname}:{wg_instance.listen_port}"
@@ -77,16 +84,19 @@ def export_firewall_configuration():
def set_instance_to_include_firewall():
firewall_settings, firewall_settings_created = FirewallSettings.objects.get_or_create(name='global')
if firewall_settings.instance_to_execute_firewall:
force_export_all_instances = False
instance_to_execute_firewall = firewall_settings.instance_to_execute_firewall
else:
force_export_all_instances = True
instance_to_execute_firewall = WireGuardInstance.objects.order_by('instance_id').first()
if instance_to_execute_firewall:
firewall_settings.instance_to_execute_firewall = instance_to_execute_firewall
firewall_settings.save()
return instance_to_execute_firewall, force_export_all_instances
instance_to_execute_firewall = firewall_settings.instance_to_execute_firewall
# The firewall script runs from the PostUp of an instance that runs on the primary server
if instance_to_execute_firewall and not instance_to_execute_firewall.hosting_worker:
return instance_to_execute_firewall, False
primary_instances = [
instance for instance in WireGuardInstance.objects.order_by('instance_id') if not instance.hosting_worker
]
instance_to_execute_firewall = primary_instances[0] if primary_instances else None
if firewall_settings.instance_to_execute_firewall != instance_to_execute_firewall:
firewall_settings.instance_to_execute_firewall = instance_to_execute_firewall
firewall_settings.save()
return instance_to_execute_firewall, True
def export_wireguard_configuration(instance_only: WireGuardInstance = None):
@@ -99,24 +109,31 @@ def export_wireguard_configuration(instance_only: WireGuardInstance = None):
else:
instances = WireGuardInstance.objects.all()
base_dir = "/etc/wireguard"
os.makedirs(base_dir, exist_ok=True)
os.makedirs(WIREGUARD_CONFIG_DIR, exist_ok=True)
os.makedirs(CLUSTER_WIREGUARD_CONFIG_DIR, exist_ok=True)
config_dirs = {instance.uuid: get_instance_config_dir(instance) for instance in instances}
firewall_inserted = False
if cleanup_orphaned:
active_instance_conf_set = {f"wg{instance.instance_id}.conf" for instance in instances}
for old_conf_file in glob.glob(os.path.join(base_dir, "wg[0-9]*.conf")):
for config_dir in (WIREGUARD_CONFIG_DIR, CLUSTER_WIREGUARD_CONFIG_DIR):
expected_conf_set = {
f"wg{instance.instance_id}.conf" for instance in instances if config_dirs[instance.uuid] == config_dir
}
if cleanup_orphaned:
orphaned_files = glob.glob(os.path.join(config_dir, "wg[0-9]*.conf"))
else:
# An exported instance may have moved between the primary server and a worker
orphaned_files = [
os.path.join(config_dir, f"wg{instance.instance_id}.conf")
for instance in instances if config_dirs[instance.uuid] != config_dir
]
for old_conf_file in orphaned_files:
filename = os.path.basename(old_conf_file)
if filename not in active_instance_conf_set:
logging.info("Removing abandoned WireGuard config: %s", filename)
if filename not in expected_conf_set and os.path.isfile(old_conf_file):
logging.info("Removing abandoned WireGuard config: %s", old_conf_file)
try:
os.remove(old_conf_file)
except FileNotFoundError:
pass
except IsADirectoryError:
continue
except PermissionError:
except (FileNotFoundError, PermissionError):
continue
for instance in instances:
@@ -213,7 +230,7 @@ def export_wireguard_configuration(instance_only: WireGuardInstance = None):
config_lines.append("")
config_content = "\n".join(config_lines)
config_path = os.path.join(base_dir, f"wg{instance.instance_id}.conf")
config_path = os.path.join(config_dirs[instance.uuid], f"wg{instance.instance_id}.conf")
with open(config_path, "w") as config_file:
config_file.write(config_content)
@@ -264,7 +281,8 @@ def download_config_or_qrcode(request):
worker_uuid = request.GET.get('worker')
server_address = None
if worker_uuid:
# In the distributed cluster mode the endpoint is always the server that runs the instance
if worker_uuid and not ClusterSettings.is_distributed():
try:
worker = Worker.objects.get(uuid=worker_uuid)
server_address = f"{worker.server_address}:{peer.wireguard_instance.listen_port}"
@@ -307,6 +325,18 @@ def restart_wireguard_interfaces(request):
config_dir = "/etc/wireguard"
interface_count = 0
error_count = 0
# Interfaces without a configuration on this server, such as instances moved to a worker
running_interfaces = subprocess.run(['wg', 'show', 'interfaces'], capture_output=True, text=True).stdout.split()
for interface_name in running_interfaces:
if re.fullmatch(r'wg[0-9]+', interface_name) and not os.path.isfile(os.path.join(config_dir, f'{interface_name}.conf')):
stop_result = subprocess.run(['ip', 'link', 'delete', interface_name], capture_output=True, text=True)
if stop_result.returncode != 0:
messages.warning(request, _("Error stopping") + f" {interface_name}|{stop_result.stderr}")
error_count += 1
else:
interface_count += 1
for filename in os.listdir(config_dir):
if filename.endswith(".conf"):
interface_name = filename[:-5]
@@ -332,8 +362,9 @@ def restart_wireguard_interfaces(request):
result = subprocess.run(reload_command, shell=True, capture_output=True, text=True)
os.remove(temp_config_path)
if result.returncode != 0:
messages.warning(request, _('Error reloading') + f" {interface_name}|{result.stderr}")
reload_error = result.stderr if result.returncode != 0 else '; '.join(sync_interface_routes(interface_name))
if reload_error:
messages.warning(request, _('Error reloading') + f" {interface_name}|{reload_error}")
error_count += 1
else:
interface_count += 1
@@ -342,11 +373,13 @@ def restart_wireguard_interfaces(request):
if not user_acl.enable_restart:
return render(request, 'access_denied.html', {'page_title': 'Access Denied'})
stop_command = f"wg-quick down {interface_name}"
stop_result = subprocess.run(stop_command, shell=True, capture_output=True, text=True)
if stop_result.returncode != 0:
messages.warning(request, _("Error stopping") + f" {interface_name}|{stop_result.stderr}")
error_count += 1
# New instances and instances moved back from a worker are not running yet
if interface_name in running_interfaces:
stop_command = f"wg-quick down {interface_name}"
stop_result = subprocess.run(stop_command, shell=True, capture_output=True, text=True)
if stop_result.returncode != 0:
messages.warning(request, _("Error stopping") + f" {interface_name}|{stop_result.stderr}")
error_count += 1
start_command = f"wg-quick up {interface_name}"
start_result = subprocess.run(start_command, shell=True, capture_output=True, text=True)
if start_result.returncode != 0: