diff --git a/api_v2/views_api.py b/api_v2/views_api.py index a75f818..4d2f3fc 100644 --- a/api_v2/views_api.py +++ b/api_v2/views_api.py @@ -13,7 +13,7 @@ from dns.models import DNSSettings, StaticHost from dns.views import export_dns_configuration from routing_templates.models import RoutingTemplate from wireguard.models import Peer, PeerAllowedIP, WireGuardInstance -from wireguard_peer.functions import func_create_new_peer +from wireguard_peer.functions import find_network_in_other_instance, func_create_new_peer from wireguard_tools.functions import func_reload_wireguard_interface from wireguard_tools.views import export_wireguard_configuration from .models import ApiKey @@ -307,6 +307,15 @@ def api_v2_manage_peer(request): announced_pairs, announced_error = _parse_ipv4_cidrs(payload.get("announced_networks")) if announced_error: return JsonResponse({"status": "error", "error_message": announced_error}, status=400) + for allowed_ip, netmask in announced_pairs or []: + other_network = find_network_in_other_instance(wireguard_instance, allowed_ip, netmask) + if other_network: + return JsonResponse({ + "status": "error", + "error_message": f"Network {allowed_ip}/{netmask} is already used by a peer of " + f"wg{other_network.peer.wireguard_instance.instance_id}. A network can be routed to " + f"only one WireGuard instance.", + }, status=400) client_route_pairs, client_route_error = _parse_ipv4_cidrs(payload.get("client_routes")) if client_route_error: diff --git a/cluster/forms.py b/cluster/forms.py index 0c9e546..c45b5dd 100644 --- a/cluster/forms.py +++ b/cluster/forms.py @@ -5,7 +5,8 @@ from django.conf import settings from django.core.exceptions import ValidationError from django.utils.translation import gettext_lazy as _ -from .models import ClusterSettings, Worker +from firewall.models import FirewallRule +from .models import DISTRIBUTED_MODES, ClusterSettings, Worker class WorkerForm(forms.ModelForm): @@ -132,4 +133,15 @@ class ClusterSettingsForm(forms.ModelForm): if cluster_enabled and not settings.WIREGUARD_STATUS_CACHE_ENABLED: raise ValidationError(_("Cluster mode requires WireGuard status cache to be enabled.")) + if cluster_enabled and cleaned_data.get('cluster_mode') in DISTRIBUTED_MODES: + instance_rules = list(FirewallRule.with_instance_interface().order_by('firewall_chain', 'sort_order')) + if instance_rules: + rule_list = ', '.join( + rule.description or f"{rule.get_firewall_chain_display()} #{rule.sort_order}" for rule in instance_rules + ) + raise ValidationError(_( + "The distributed cluster mode does not allow firewall rules bound to a specific WireGuard instance. " + "Change them to wg+ or IP addresses first: %(rules)s" + ) % {'rules': rule_list}) + return cleaned_data \ No newline at end of file diff --git a/cluster/migrations/0019_alter_clustersettings_cluster_mode.py b/cluster/migrations/0019_alter_clustersettings_cluster_mode.py new file mode 100644 index 0000000..eef9fd3 --- /dev/null +++ b/cluster/migrations/0019_alter_clustersettings_cluster_mode.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.19 on 2026-11-20 19:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('cluster', '0018_cluster_v2'), + ] + + operations = [ + migrations.AlterField( + model_name='clustersettings', + name='cluster_mode', + field=models.CharField(choices=[('mirror', 'Mirror'), ('distributed', 'Distributed')], default='mirror', max_length=16), + ), + ] diff --git a/cluster/models.py b/cluster/models.py index 81069d0..4e96cbd 100644 --- a/cluster/models.py +++ b/cluster/models.py @@ -3,12 +3,16 @@ import uuid from django.db import models from django.utils.translation import gettext_lazy as _ +DISTRIBUTED_MODES = ('distributed',) + class ClusterSettings(models.Model): name = models.CharField(default='cluster_settings', max_length=16, unique=True) enabled = models.BooleanField(default=False) primary_enable_wireguard = models.BooleanField(default=True) - cluster_mode = models.CharField(default='mirror', max_length=16, choices=(('mirror', 'Mirror'), )) + cluster_mode = models.CharField( + default='mirror', max_length=16, choices=(('mirror', 'Mirror'), ('distributed', 'Distributed')) + ) worker_display = models.CharField( default='server_address', max_length=16, choices=( ('name', 'Name'), ('server_address', 'Server Address'), @@ -27,6 +31,11 @@ class ClusterSettings(models.Model): def cluster_enabled(cls): return cls.objects.filter(name='cluster_settings', enabled=True).exists() + @classmethod + def is_distributed(cls): + """Each WireGuard instance runs on a single server. Only applies while the cluster is enabled.""" + return cls.objects.filter(name='cluster_settings', enabled=True, cluster_mode__in=DISTRIBUTED_MODES).exists() + class Worker(models.Model): name = models.CharField(max_length=100, unique=True) diff --git a/cluster/sync.py b/cluster/sync.py index 6ddf0f8..9208cf8 100644 --- a/cluster/sync.py +++ b/cluster/sync.py @@ -11,12 +11,13 @@ from django.utils.dateparse import parse_datetime from app_gateway.caddy_config_export import ( add_certificate_files, certificate_file_name, load_worker_export, read_exported_certificate ) -from cluster.models import WorkerStatus +from cluster.models import ClusterSettings, WorkerStatus from dns.functions import generate_dnsmasq_config from dns.models import DNSFilterList from firewall.tools import generate_firewall_script +from wireguard.models import WireGuardInstance +from wireguard_tools.functions import CLUSTER_WIREGUARD_CONFIG_DIR, WIREGUARD_CONFIG_DIR -WIREGUARD_CONFIG_DIR = '/etc/wireguard' DNSMASQ_CONFIG_DIR = '/etc/dnsmasq' DNSMASQ_MAIN_CONFIG = 'wireguard_webadmin_dns.conf' @@ -67,7 +68,17 @@ class Section: def build_wireguard_section(worker): section = Section() - for config_path in sorted(glob.glob(os.path.join(WIREGUARD_CONFIG_DIR, 'wg[0-9]*.conf'))): + if ClusterSettings.is_distributed(): + # Only the instances that run on this worker + config_paths = [ + os.path.join(CLUSTER_WIREGUARD_CONFIG_DIR, f'wg{instance_id}.conf') + for instance_id in WireGuardInstance.objects.filter(worker=worker).values_list('instance_id', flat=True) + ] + else: + config_paths = glob.glob(os.path.join(WIREGUARD_CONFIG_DIR, 'wg[0-9]*.conf')) + for config_path in sorted(config_paths): + if not os.path.isfile(config_path): + continue content, content_sha256 = _read_file(config_path) section.add_file(os.path.basename(config_path), content, content_sha256) return section @@ -75,7 +86,7 @@ def build_wireguard_section(worker): def build_firewall_section(worker): section = Section() - section.add_file('wg-firewall.sh', generate_firewall_script(cluster_node=True)) + section.add_file('wg-firewall.sh', generate_firewall_script(cluster_node=True, worker=worker)) return section diff --git a/cluster/tests.py b/cluster/tests.py index 4937496..1e5bb16 100644 --- a/cluster/tests.py +++ b/cluster/tests.py @@ -3,6 +3,7 @@ import os import tempfile from unittest.mock import patch +from django.contrib.auth.models import User from django.test import TestCase, override_settings from django.utils import timezone @@ -12,9 +13,16 @@ from app_gateway import caddy_config_export from app_gateway.caddy_config_export import certificate_file_name, export_caddy_config from app_gateway.models import Application, ApplicationHost, DnsChallengeCredential from cluster import sync +from cluster.forms import ClusterSettingsForm from cluster.models import ClusterSettings, Worker, WorkerStatus from dns.models import DNSSettings +from firewall.forms import FirewallRuleForm +from firewall.models import FirewallRule, FirewallSettings, RedirectRule from firewall.tools import generate_firewall_script +from user_manager.models import UserAcl +from wireguard.models import Peer, PeerAllowedIP, WireGuardInstance +from wireguard_tools.functions import get_config_servers +from wireguard_tools.views import export_wireguard_configuration, generate_peer_config FAKE_CERTIFICATE = '-----BEGIN CERTIFICATE-----\nfake\n-----END CERTIFICATE-----\n' FAKE_PRIVATE_KEY = '-----BEGIN PRIVATE KEY-----\nfake\n-----END PRIVATE KEY-----\n' @@ -271,3 +279,159 @@ class ClusterTrafficAggregationTests(TestCase): self.assertEqual(combined['PEER']['endpoints'], 'node:1') self.assertEqual(combined['PEER']['source_name'], 'worker_node-a') self.assertEqual(combined['NODE_ONLY']['transfer'], {'tx': 7, 'rx': 3}) + + +def create_instance(instance_id, worker=None): + return WireGuardInstance.objects.create( + instance_id=instance_id, private_key=f'private-{instance_id}', public_key=f'public-{instance_id}', + hostname='primary.example.test', listen_port=51820 + instance_id, address=f'10.188.{instance_id}.1', + worker=worker, + ) + + +@override_settings(WIREGUARD_STATUS_CACHE_ENABLED=True) +class DistributedModeTests(ClusterTestMixin, TestCase): + """wg0 runs on the primary server, wg1 on node-a and node-b runs no instance.""" + + def setUp(self): + super().setUp() + self.cluster_wireguard_dir = os.path.join(self.wireguard_dir, 'cluster') + for target, value in ( + ('cluster.sync.CLUSTER_WIREGUARD_CONFIG_DIR', self.cluster_wireguard_dir), + ('wireguard_tools.functions.WIREGUARD_CONFIG_DIR', self.wireguard_dir), + ('wireguard_tools.functions.CLUSTER_WIREGUARD_CONFIG_DIR', self.cluster_wireguard_dir), + ('wireguard_tools.views.WIREGUARD_CONFIG_DIR', self.wireguard_dir), + ('wireguard_tools.views.CLUSTER_WIREGUARD_CONFIG_DIR', self.cluster_wireguard_dir), + ): + patcher = patch(target, value) + patcher.start() + self.addCleanup(patcher.stop) + + self.worker_b = Worker.objects.create(name='node-b', hostname='node-b.example.test') + self.primary_instance = create_instance(0) + self.worker_instance = create_instance(1, worker=self.worker) + ClusterSettings.objects.filter(name='cluster_settings').update(cluster_mode='distributed') + + def set_mode(self, enabled=True, cluster_mode='distributed'): + ClusterSettings.objects.filter(name='cluster_settings').update(enabled=enabled, cluster_mode=cluster_mode) + + def create_peer(self, instance): + peer = Peer.objects.create( + name=f'peer-wg{instance.instance_id}', public_key=f'peer-public-{instance.instance_id}', + pre_shared_key='', private_key=f'peer-private-{instance.instance_id}', wireguard_instance=instance, + ) + PeerAllowedIP.objects.create( + peer=peer, priority=0, allowed_ip=f'10.188.{instance.instance_id}.10', netmask=32, config_file='server' + ) + return peer + + def test_hosting_worker_only_in_distributed_mode(self): + self.assertTrue(ClusterSettings.is_distributed()) + self.assertEqual(self.worker_instance.hosting_worker, self.worker) + self.assertIsNone(self.primary_instance.hosting_worker) + + self.set_mode(cluster_mode='mirror') + self.assertIsNone(WireGuardInstance.objects.get(instance_id=1).hosting_worker) + self.set_mode(enabled=False) + self.assertFalse(ClusterSettings.is_distributed()) + self.assertIsNone(WireGuardInstance.objects.get(instance_id=1).hosting_worker) + + def test_export_places_configs_by_server(self): + FirewallSettings.objects.update_or_create(name='global', defaults={'instance_to_execute_firewall': self.worker_instance}) + export_wireguard_configuration() + self.assertEqual(sorted(os.listdir(self.cluster_wireguard_dir)), ['wg1.conf']) + self.assertIn('wg0.conf', os.listdir(self.wireguard_dir)) + self.assertNotIn('wg1.conf', os.listdir(self.wireguard_dir)) + # The firewall script must run from an instance of the primary server + self.assertEqual(FirewallSettings.objects.get(name='global').instance_to_execute_firewall, self.primary_instance) + with open(os.path.join(self.wireguard_dir, 'wg0.conf'), encoding='utf-8') as config_file: + self.assertIn('PostUp = /etc/wireguard/wg-firewall.sh', config_file.read()) + + # Moving a single instance back to the primary server removes it from the cluster directory + WireGuardInstance.objects.filter(instance_id=1).update(worker=None) + export_wireguard_configuration(WireGuardInstance.objects.get(instance_id=1)) + self.assertIn('wg1.conf', os.listdir(self.wireguard_dir)) + self.assertEqual(os.listdir(self.cluster_wireguard_dir), []) + + def test_wireguard_section_only_has_worker_instances(self): + export_wireguard_configuration() + self.assertEqual(sorted(sync.build_section('wireguard', self.worker).files), ['wg1.conf']) + self.assertEqual(sync.build_section('wireguard', self.worker_b).files, {}) + + self.set_mode(cluster_mode='mirror') + export_wireguard_configuration() + self.assertEqual(sorted(sync.build_section('wireguard', self.worker_b).files), ['wg0.conf', 'wg1.conf']) + + def test_port_forwards_only_on_instance_server(self): + RedirectRule.objects.create(port=8081, wireguard_instance=self.worker_instance, ip_address='10.188.1.10') + RedirectRule.objects.create(port=8082, wireguard_instance=self.primary_instance, ip_address='10.188.0.10') + worker_script = generate_firewall_script(cluster_node=True, worker=self.worker) + other_worker_script = generate_firewall_script(cluster_node=True, worker=self.worker_b) + primary_script = generate_firewall_script() + + self.assertIn('--dport 8081', worker_script) + self.assertNotIn('--dport 8082', worker_script) + self.assertNotIn('--dport 808', other_worker_script) + self.assertIn('--dport 8082', primary_script) + self.assertNotIn('--dport 8081', primary_script) + + self.set_mode(cluster_mode='mirror') + self.assertIn('--dport 8082', generate_firewall_script(cluster_node=True, worker=self.worker_b)) + + def test_activation_blocked_by_instance_interface_rules(self): + self.set_mode(cluster_mode='mirror') + FirewallRule.objects.create(description='wg0 to internet', in_interface='wg0', out_interface='eth0') + cluster_settings = ClusterSettings.objects.get(name='cluster_settings') + data = {'enabled': True, 'primary_enable_wireguard': True, 'cluster_mode': 'distributed', 'worker_display': 'name'} + + form = ClusterSettingsForm(data=data, instance=cluster_settings) + self.assertFalse(form.is_valid()) + self.assertIn('wg0 to internet', str(form.non_field_errors())) + + FirewallRule.objects.update(in_interface='wg+') + self.assertTrue(ClusterSettingsForm(data=data, instance=cluster_settings).is_valid()) + + def test_firewall_rule_form_rejects_instance_interface(self): + def rule_form(in_interface): + return FirewallRuleForm(data={ + 'firewall_chain': 'forward', 'in_interface': in_interface, 'out_interface': '', 'rule_action': 'accept', + 'source_netmask': 32, 'destination_netmask': 32, 'protocol': '', 'sort_order': 10, + }) + + form = rule_form('wg0') + self.assertFalse(form.is_valid()) + self.assertIn('distributed cluster mode', str(form.non_field_errors())) + self.assertTrue(rule_form('wg+').is_valid()) + + self.set_mode(cluster_mode='mirror') + self.assertTrue(rule_form('wg0').is_valid()) + + def test_peer_config_uses_instance_server(self): + worker_peer = self.create_peer(self.worker_instance) + primary_peer = self.create_peer(self.primary_instance) + + self.assertIn('Endpoint = node-a.example.test:51821', generate_peer_config(worker_peer.uuid)) + # A server picked for another worker is ignored: the instance only runs on node-a + self.assertIn( + 'Endpoint = node-a.example.test:51821', + generate_peer_config(worker_peer.uuid, server_address='node-b.example.test:51821'), + ) + self.assertIn('Endpoint = primary.example.test:51820', generate_peer_config(primary_peer.uuid)) + + self.assertEqual(get_config_servers(self.worker_instance), [{'name': 'node-a.example.test', 'uuid': str(self.worker.uuid)}]) + self.assertEqual([server['uuid'] for server in get_config_servers(self.primary_instance)], ['']) + + self.set_mode(cluster_mode='mirror') + self.assertEqual(len(get_config_servers(self.worker_instance)), 3) + + def test_worker_with_instances_cannot_be_deleted(self): + user = User.objects.create_user(username='cluster-admin', password='unused-test-password') + UserAcl.objects.create(user=user, user_level=50) + self.client.force_login(user) + + response = self.client.get(f'/cluster/worker/manage/?uuid={self.worker.uuid}&action=delete&confirmation=delete') + self.assertRedirects(response, f'/cluster/worker/view/?uuid={self.worker.uuid}', fetch_redirect_response=False) + self.assertTrue(Worker.objects.filter(pk=self.worker.pk).exists()) + + self.client.get(f'/cluster/worker/manage/?uuid={self.worker_b.uuid}&action=delete&confirmation=delete') + self.assertFalse(Worker.objects.filter(pk=self.worker_b.pk).exists()) diff --git a/cluster/views.py b/cluster/views.py index d850512..6fb0890 100644 --- a/cluster/views.py +++ b/cluster/views.py @@ -1,6 +1,7 @@ from django.conf import settings from django.contrib import messages from django.contrib.auth.decorators import login_required +from django.db.models import ProtectedError from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse from django.utils.dateparse import parse_datetime @@ -9,6 +10,8 @@ from django.utils.translation import gettext_lazy as _ from api.models import WireguardStatusCache from app_gateway.setup_defaults import get_webadmin_hostnames from user_manager.models import UserAcl +from wireguard.models import WireGuardInstance +from wireguard_tools.views import export_firewall_configuration, export_wireguard_configuration from .forms import WorkerForm, ClusterSettingsForm from .models import ClusterSettings, Worker, WorkerStatus from .sync import SECTION_NAMES, get_worker_sync_state, get_worker_warnings @@ -39,6 +42,10 @@ def describe_worker(worker): worker.services = sorted((worker_status.services_status or {}).items()) if worker_status else [] worker.runs_gateway = any(service == 'caddy' for service, state in worker.services) worker.outdated = bool(worker_status and worker_status.worker_version != settings.CLUSTER_WORKER_CURRENT_VERSION) + # WireGuard instances that run only on this worker (distributed cluster mode) + worker.hosted_instances = ( + list(worker.wireguard_instances.order_by('instance_id')) if ClusterSettings.is_distributed() else [] + ) return worker @@ -63,6 +70,12 @@ def cluster_main(request): 'workers': workers, 'cluster_settings': cluster_settings, 'current_worker_version': settings.CLUSTER_WORKER_CURRENT_VERSION, + 'distributed': ClusterSettings.is_distributed(), + 'primary_instances': WireGuardInstance.objects.filter(worker__isnull=True).order_by('instance_id'), + # Instances that no server runs: their worker is disabled + 'stopped_instances': [ + instance for worker in workers if not worker.enabled for instance in worker.hosted_instances + ], 'summary': { 'enabled': len(enabled_workers), 'online': sum(1 for worker in enabled_workers if worker.operational), @@ -119,6 +132,7 @@ def worker_details(request): 'worker': worker, 'cluster_settings': cluster_settings, 'current_worker_version': settings.CLUSTER_WORKER_CURRENT_VERSION, + 'distributed': ClusterSettings.is_distributed(), 'sections': sections, 'certificates': certificates, 'primary_address': primary_address, @@ -140,8 +154,14 @@ def worker_manage(request): if request.GET.get('action') == 'delete': worker_name = worker.name if request.GET.get('confirmation') == 'delete': - worker.delete() - messages.success(request, _('Worker deleted|Worker deleted: ') + worker_name) + try: + worker.delete() + messages.success(request, _('Worker deleted|Worker deleted: ') + worker_name) + except ProtectedError: + messages.warning(request, _( + 'Worker not deleted|Move the WireGuard instances of this worker to another server first: ' + ) + ', '.join(str(instance) for instance in worker.wireguard_instances.all())) + return redirect(reverse('worker_details') + f'?uuid={worker.uuid}') else: messages.warning(request, _('Worker not deleted|Invalid confirmation.')) return redirect(reverse('cluster_main')) @@ -195,10 +215,17 @@ def cluster_settings(request): cluster_settings, cluster_settings_created = ClusterSettings.objects.get_or_create(name='cluster_settings') + was_distributed = ClusterSettings.is_distributed() form = ClusterSettingsForm(request.POST or None, instance=cluster_settings) if form.is_valid(): - form.save() + cluster_settings = form.save() messages.success(request, _('Cluster settings updated successfully.')) + if ClusterSettings.is_distributed() != was_distributed: + # Where each instance runs may have changed: workers synchronize the new files right away and the + # primary server moves its interfaces when the configuration is applied + export_wireguard_configuration() + export_firewall_configuration() + WireGuardInstance.objects.update(pending_changes=True) return redirect(reverse('cluster_main')) form_description = { @@ -212,6 +239,7 @@ def cluster_settings(request):
Cluster Mode

Mirror: every worker receives the same WireGuard instances, peers, firewall and DNS configuration. Peers can connect to any server.

+

Distributed: each WireGuard instance runs only on the server selected in the instance settings (the primary server by default). Peers connect to that server. Firewall rules cannot be bound to a specific instance interface: use wg+ or IP addresses. After changing the mode, apply the WireGuard configuration so the primary server stops the instances that moved to workers.

Worker Display

How workers are named in server selections (peer configuration downloads and VPN invites).

diff --git a/containers/cluster_node/cluster_worker.py b/containers/cluster_node/cluster_worker.py index 49f09ea..aebe4b4 100644 --- a/containers/cluster_node/cluster_worker.py +++ b/containers/cluster_node/cluster_worker.py @@ -1,4 +1,5 @@ import glob +import ipaddress import json import logging import os @@ -17,7 +18,7 @@ logger = logging.getLogger(__name__) MASTER_SERVER_ADDRESS = os.environ.get('MASTER_SERVER_ADDRESS', '') TOKEN = os.environ.get('TOKEN', '') -WORKER_VERSION = 12 +WORKER_VERSION = 13 REQUEST_TIMEOUT = 30 MANIFEST_INTERVAL = 60 DEFAULT_STATS_INTERVAL = 60 @@ -57,6 +58,48 @@ def interface_is_up(interface): return run_command(['wg', 'show', interface], check=False).returncode == 0 +def sync_interface_routes(interface): + """ + Bring the IPv4 routes of a running interface in line with its allowed IPs, as wg-quick does on start. + wg syncconf only updates the peers. Default routes (/0) are left to wg-quick. + """ + allowed_networks = set() + for line in run_command(['wg', 'show', interface, 'allowed-ips']).stdout.splitlines(): + for value in line.split()[1:]: + try: + network = ipaddress.IPv4Network(value, strict=False) + except ValueError: + continue + if network.prefixlen: + allowed_networks.add(network) + allowed_networks = sorted(allowed_networks, key=lambda network: network.prefixlen) + + errors = [] + for network in allowed_networks: + if run_command(['ip', '-4', 'route', 'show', 'dev', interface, 'match', str(network)], check=False).stdout.strip(): + continue + # replace instead of add: the network may still be routed to another instance it was moved from + result = run_command(['ip', '-4', 'route', 'replace', str(network), 'dev', interface], check=False) + if result.returncode != 0: + errors.append(f'{network}: {result.stderr.strip()}') + + for route in run_command(['ip', '-o', '-4', 'route', 'show', 'dev', interface], check=False).stdout.splitlines(): + fields = route.split() + if not fields or 'kernel' in fields: + continue + try: + route_network = ipaddress.IPv4Network(fields[0], strict=False) + except ValueError: + continue + # Only routes that no allowed IP covers are removed: the interface drops their traffic anyway + if any(route_network.subnet_of(network) for network in allowed_networks): + continue + result = run_command(['ip', '-4', 'route', 'delete', str(route_network), 'dev', interface], check=False) + if result.returncode != 0: + errors.append(f'{route_network}: {result.stderr.strip()}') + if errors: + raise SectionError(f"{interface} routes: {'; '.join(errors)}") + def interface_section(config): """Return the [Interface] part of a WireGuard config, used to decide between syncconf and restart.""" return config.split('[Peer]', 1)[0].strip() @@ -247,6 +290,9 @@ class ClusterNodeAgent: logger.info('Synchronized peers of WireGuard interface %s', interface) except SectionError as error: errors.append(str(error)) + # Back to the content actually applied, so the next attempt sees the change again + if previous_content is not None: + write_file(config_path, previous_content) self.wireguard_active = True if errors: @@ -261,6 +307,7 @@ class ClusterNodeAgent: run_command(['wg', 'syncconf', interface, stripped_file.name]) finally: os.remove(stripped_file.name) + sync_interface_routes(interface) def apply_dns(self, section): files = section['files'] diff --git a/firewall/forms.py b/firewall/forms.py index 0ed2788..d58f771 100644 --- a/firewall/forms.py +++ b/firewall/forms.py @@ -3,7 +3,8 @@ import re from django import forms from django.utils.translation import gettext_lazy as _ -from firewall.models import FirewallRule, FirewallSettings, RedirectRule +from cluster.models import ClusterSettings +from firewall.models import INSTANCE_INTERFACE_REGEX, FirewallRule, FirewallSettings, RedirectRule from wgwadmlibrary.tools import list_network_interfaces from wireguard.models import Peer, WireGuardInstance @@ -143,9 +144,18 @@ class FirewallRuleForm(forms.ModelForm): firewall_chain = cleaned_data.get('firewall_chain') rule_action = cleaned_data.get('rule_action') in_interface = cleaned_data.get('in_interface') + out_interface = cleaned_data.get('out_interface') destination_port = self.cleaned_data.get('destination_port') protocol = cleaned_data.get('protocol') + if ClusterSettings.is_distributed() and any( + interface and re.match(INSTANCE_INTERFACE_REGEX, interface) for interface in (in_interface, out_interface) + ): + raise forms.ValidationError(_( + "The distributed cluster mode does not allow rules bound to a specific WireGuard instance. " + "Use wg+ or IP addresses instead." + )) + if firewall_chain == 'forward' and rule_action not in ['accept', 'drop', 'reject']: raise forms.ValidationError("Invalid rule action for firewall chain 'forward'. Allowed actions are 'accept', 'drop', and 'reject'.") diff --git a/firewall/models.py b/firewall/models.py index 0f19536..4c2d844 100644 --- a/firewall/models.py +++ b/firewall/models.py @@ -1,11 +1,15 @@ import uuid from django.db import models +from django.db.models import Q from django.utils.translation import gettext_lazy as _ from wireguard.models import NETMASK_CHOICES from wireguard.models import Peer, WireGuardInstance +# Interface of a single WireGuard instance (wg0, wg1...). wg+ matches every instance. +INSTANCE_INTERFACE_REGEX = r'^wg[0-9]+$' + class RedirectRule(models.Model): description = models.CharField(max_length=100, blank=True, null=True) @@ -69,6 +73,13 @@ class FirewallRule(models.Model): def __str__(self): return str(self.uuid) + @classmethod + def with_instance_interface(cls): + """Rules bound to the interface of a specific instance, not allowed in the distributed cluster mode.""" + return cls.objects.filter( + Q(in_interface__regex=INSTANCE_INTERFACE_REGEX) | Q(out_interface__regex=INSTANCE_INTERFACE_REGEX) + ) + class FirewallSettings(models.Model): name = models.CharField(max_length=6, default='global', unique=True) diff --git a/firewall/tools.py b/firewall/tools.py index dec4591..c615366 100644 --- a/firewall/tools.py +++ b/firewall/tools.py @@ -3,6 +3,7 @@ from django.db.models import Q, Prefetch from django.utils import timezone from django.utils.translation import gettext_lazy as _ +from cluster.models import ClusterSettings from firewall.models import FirewallRule, FirewallSettings, RedirectRule from wireguard.models import PeerAllowedIP, WireGuardInstance, Peer @@ -281,12 +282,17 @@ def generate_firewall_footer(): return footer -def generate_port_forward_firewall(server_hostname=PRIMARY_HOSTNAME): +def generate_port_forward_firewall(server_hostname=PRIMARY_HOSTNAME, worker=None): firewall_settings, firewall_settings_created = FirewallSettings.objects.get_or_create(name='global') redirect_firewall = '' wan_interface = firewall_settings.wan_interface + distributed = ClusterSettings.is_distributed() + server_worker_id = worker.id if worker else None - for redirect_rule in RedirectRule.objects.all().order_by('port'): + for redirect_rule in RedirectRule.objects.select_related('wireguard_instance').order_by('port'): + # In the distributed cluster mode only the server that runs the instance forwards the port + if distributed and redirect_rule.wireguard_instance and redirect_rule.wireguard_instance.worker_id != server_worker_id: + continue description = f" - {redirect_rule.description} " if redirect_rule.description else "" rule_destination = redirect_rule.ip_address try: @@ -319,16 +325,17 @@ def generate_port_forward_firewall(server_hostname=PRIMARY_HOSTNAME): return redirect_firewall -def generate_firewall_script(cluster_node=False): +def generate_firewall_script(cluster_node=False, worker=None): """ Build wg-firewall.sh for the primary server or for a cluster node. On cluster nodes dnsmasq shares the network namespace of the node, so DNS is answered directly on the instance address without DNAT. + The worker selects the port forwards of the instances it runs in the distributed cluster mode. """ firewall_content = generate_firewall_header(cluster_node=cluster_node) if not cluster_node: firewall_content += generate_redirect_dns_rules() firewall_content += generate_port_forward_firewall( - server_hostname=CLUSTER_NODE_HOSTNAME if cluster_node else PRIMARY_HOSTNAME + server_hostname=CLUSTER_NODE_HOSTNAME if cluster_node else PRIMARY_HOSTNAME, worker=worker ) firewall_content += generate_route_policy_rules() firewall_content += export_user_firewall() diff --git a/templates/cluster/worker_details.html b/templates/cluster/worker_details.html index b68b294..ee0c770 100644 --- a/templates/cluster/worker_details.html +++ b/templates/cluster/worker_details.html @@ -48,6 +48,16 @@
{% if worker.status %}{{ worker.status.last_seen|date:"SHORT_DATETIME_FORMAT" }}{% else %}{% trans 'Never' %}{% endif %}
{% trans 'Version' %}
{% if worker.status %}v{{ worker.status.worker_version }}{% else %}-{% endif %} ({% trans 'current' %}: v{{ current_worker_version }})
+ {% if distributed %} +
{% trans 'WireGuard instances' %}
+
+ {% for instance in worker.hosted_instances %} + wg{{ instance.instance_id }}{% if instance.name %} · {{ instance.name }}{% endif %} + {% empty %} + - + {% endfor %} +
+ {% endif %}
{% trans 'App Gateway' %}
{% if worker.runs_gateway %}{% trans 'Enabled' %}{% else %}{% trans 'Not installed' %}{% endif %}
diff --git a/templates/cluster/workers_list.html b/templates/cluster/workers_list.html index 027a68e..32c8c00 100644 --- a/templates/cluster/workers_list.html +++ b/templates/cluster/workers_list.html @@ -9,6 +9,14 @@ {% trans 'Cluster Settings' %} {% endif %} +{% if stopped_instances %} +
+ + {% trans 'These WireGuard instances are offline because their worker is disabled:' %} + {% for instance in stopped_instances %}wg{{ instance.instance_id }} ({{ instance.worker }}){% if not forloop.last %}, {% endif %}{% endfor %}. + {% trans 'Enable the worker or move the instances to another server.' %} +
+{% endif %} {% if not cache_enabled %}
@@ -51,6 +59,11 @@
{% trans 'Primary WireGuard' %} {% if cluster_settings.primary_enable_wireguard %}{% trans 'Enabled' %}{% else %}{% trans 'Disabled' %}{% endif %} + {% if distributed %} + + {% for instance in primary_instances %}wg{{ instance.instance_id }}{% if not forloop.last %}, {% endif %}{% empty %}{% trans 'No WireGuard instances' %}{% endfor %} + + {% endif %}
@@ -88,6 +101,15 @@ {% if worker.ip_lock %}{% endif %} {% if worker.city or worker.country %} · {{ worker.city|default:'' }}{% if worker.city and worker.country %}, {% endif %}{{ worker.country|default:'' }}{% endif %} + {% if distributed %} +
+ {% for instance in worker.hosted_instances %} + wg{{ instance.instance_id }}{% if instance.name %} · {{ instance.name }}{% endif %} + {% empty %} + {% trans 'No WireGuard instances' %} + {% endfor %} + + {% endif %} {% if not worker.enabled %} diff --git a/vpn_invite_public/views.py b/vpn_invite_public/views.py index a3e12b6..dd1ad03 100644 --- a/vpn_invite_public/views.py +++ b/vpn_invite_public/views.py @@ -2,8 +2,9 @@ from django.http import Http404 from django.shortcuts import render from django.utils import timezone -from cluster.models import ClusterSettings, Worker +from cluster.models import ClusterSettings from vpn_invite.models import PeerInvite, InviteSettings +from wireguard_tools.functions import get_config_servers def view_public_vpn_invite(request): @@ -15,13 +16,7 @@ def view_public_vpn_invite(request): raise Http404 cluster_settings = ClusterSettings.objects.filter(name='cluster_settings', enabled=True).first() - servers = [] - if cluster_settings: - if cluster_settings.primary_enable_wireguard: - servers.append({'name': 'Primary Server', 'uuid': ''}) - - for worker in Worker.objects.filter(enabled=True): - servers.append({'name': worker.display_name, 'uuid': str(worker.uuid)}) + servers = get_config_servers(peer_invite.peer.wireguard_instance) context = { 'peer_invite': peer_invite, diff --git a/wireguard/forms.py b/wireguard/forms.py index 844cc9e..7f76ca7 100644 --- a/wireguard/forms.py +++ b/wireguard/forms.py @@ -8,6 +8,7 @@ from django.conf import settings from django.urls import reverse_lazy from django.utils.translation import gettext_lazy as _ +from cluster.models import ClusterSettings, Worker from wgwadmlibrary.tools import is_valid_ip_or_hostname from .models import NETMASK_CHOICES, WireGuardInstance @@ -34,7 +35,8 @@ class WireGuardInstanceForm(forms.ModelForm): model = WireGuardInstance fields = [ 'name', 'instance_id', 'private_key', 'public_key','hostname', 'listen_port', 'address', - 'netmask', 'peer_list_refresh_interval', 'dns_primary', 'dns_secondary', 'dns_search_domains', 'enforce_route_policy' + 'netmask', 'peer_list_refresh_interval', 'dns_primary', 'dns_secondary', 'dns_search_domains', 'enforce_route_policy', + 'worker' ] def __init__(self, *args, **kwargs): @@ -50,6 +52,19 @@ class WireGuardInstanceForm(forms.ModelForm): self.fields['peer_list_refresh_interval'].widget.attrs['readonly'] = True self.fields['private_key'].widget = forms.PasswordInput(attrs={'class': 'form-control'}, render_value=True) + # The server that runs the instance is only chosen in the distributed cluster mode + distributed = ClusterSettings.is_distributed() + if distributed: + self.fields['worker'].queryset = Worker.objects.order_by('name') + self.fields['worker'].label = _('Server') + self.fields['worker'].empty_label = _('Primary Server') + self.fields['worker'].help_text = _( + 'The instance runs only on this server. Peer configurations of instances on a worker use the ' + 'hostname of the worker as endpoint.' + ) + else: + del self.fields['worker'] + self.helper = FormHelper() self.helper.form_method = 'post' @@ -100,6 +115,10 @@ class WireGuardInstanceForm(forms.ModelForm): Column('enforce_route_policy', css_class='form-group col-md-12 mb-0'), css_class='form-row' ), + Row( + Column('worker', css_class='form-group col-md-12 mb-0'), + css_class='form-row' + ) if distributed else HTML(''), css_class='col-lg-12' ), css_class='row' diff --git a/wireguard/migrations/0034_wireguardinstance_worker.py b/wireguard/migrations/0034_wireguardinstance_worker.py new file mode 100644 index 0000000..5700953 --- /dev/null +++ b/wireguard/migrations/0034_wireguardinstance_worker.py @@ -0,0 +1,20 @@ +# Generated by Django 5.2.19 on 2026-11-20 19:13 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('cluster', '0019_alter_clustersettings_cluster_mode'), + ('wireguard', '0033_wireguardinstance_dns_search_domains'), + ] + + operations = [ + migrations.AddField( + model_name='wireguardinstance', + name='worker', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='wireguard_instances', to='cluster.worker'), + ), + ] diff --git a/wireguard/models.py b/wireguard/models.py index ac7d5f9..763aa62 100644 --- a/wireguard/models.py +++ b/wireguard/models.py @@ -4,6 +4,7 @@ from typing import Optional from django.db import models +from cluster.models import ClusterSettings from wireguard_tools.networks import normalize_cidr_list, normalize_cidr_pairs, safe_network_cidr NETMASK_CHOICES = ( @@ -68,6 +69,10 @@ class WireGuardInstance(models.Model): pending_changes = models.BooleanField(default=True) legacy_firewall = models.BooleanField(default=False) enforce_route_policy = models.BooleanField(default=False) + # Server that runs the instance in the distributed cluster mode. Empty means the primary server. + worker = models.ForeignKey( + 'cluster.Worker', on_delete=models.PROTECT, blank=True, null=True, related_name='wireguard_instances' + ) created = models.DateTimeField(auto_now_add=True) updated = models.DateTimeField(auto_now=True) @@ -79,6 +84,13 @@ class WireGuardInstance(models.Model): else: return 'wg' + str(self.instance_id) + @property + def hosting_worker(self): + """Worker that runs this instance, or None when it runs on the primary server.""" + if self.worker_id and ClusterSettings.is_distributed(): + return self.worker + return None + @property def network_cidr(self): return safe_network_cidr(self.address, self.netmask) diff --git a/wireguard_peer/forms.py b/wireguard_peer/forms.py index fe9deda..f76e0ec 100644 --- a/wireguard_peer/forms.py +++ b/wireguard_peer/forms.py @@ -13,6 +13,7 @@ from django.utils.translation import gettext_lazy as _ from scheduler.models import PeerScheduling from wireguard.models import NETMASK_CHOICES, Peer, PeerAllowedIP +from wireguard_peer.functions import find_network_in_other_instance class PeerModelForm(forms.ModelForm): @@ -101,6 +102,13 @@ class PeerAllowedIPForm(forms.ModelForm): else: if ipaddress.ip_address(allowed_ip) in wireguard_network: raise forms.ValidationError(_("The IP address belongs to the Peer's WireGuard instance network range. Please check the IP address or change use priority 0 instead.")) + other_network = find_network_in_other_instance(self.current_peer.wireguard_instance, allowed_ip, netmask) + if other_network: + raise forms.ValidationError( + _("This network is already used by peer %(peer)s of wg%(instance)s. A network can be routed to only one WireGuard instance.") % { + 'peer': other_network.peer, 'instance': other_network.peer.wireguard_instance.instance_id + } + ) elif self.config_file == 'client': if self.current_peer.routing_template and not self.current_peer.routing_template.allow_peer_custom_routes: raise forms.ValidationError(_("The peer's routing template does not allow custom routes.")) diff --git a/wireguard_peer/functions.py b/wireguard_peer/functions.py index a377df4..218f345 100644 --- a/wireguard_peer/functions.py +++ b/wireguard_peer/functions.py @@ -1,3 +1,4 @@ +import ipaddress import subprocess from typing import Any, Dict, Optional, Tuple @@ -7,6 +8,27 @@ from routing_templates.models import RoutingTemplate from wireguard.models import Peer, PeerAllowedIP, WireGuardInstance +def find_network_in_other_instance(wireguard_instance, allowed_ip, netmask): + """ + Server side network of a peer of another instance that is the same as the given one. Each instance routes its + peer networks to its own interface, so the same network on two instances cannot be routed: the second interface + fails to start. + """ + try: + network = ipaddress.ip_network(f'{allowed_ip}/{netmask}', strict=False) + except ValueError: + return None + other_networks = PeerAllowedIP.objects.filter(config_file='server').exclude( + peer__wireguard_instance=wireguard_instance + ).select_related('peer__wireguard_instance') + for other_network in other_networks: + try: + if ipaddress.ip_network(f'{other_network.allowed_ip}/{other_network.netmask}', strict=False) == network: + return other_network + except ValueError: + continue + return None + def generate_peer_default(wireguard_instance): private_key = subprocess.check_output('wg genkey', shell=True).decode('utf-8').strip() public_key = subprocess.check_output(f'echo {private_key} | wg pubkey', shell=True).decode('utf-8').strip() diff --git a/wireguard_peer/tests.py b/wireguard_peer/tests.py index 7ce503c..6cee853 100644 --- a/wireguard_peer/tests.py +++ b/wireguard_peer/tests.py @@ -1,3 +1,98 @@ +import json + from django.test import TestCase -# Create your tests here. +from api_v2.models import ApiKey +from wireguard.models import Peer, PeerAllowedIP, WireGuardInstance +from wireguard_peer.forms import PeerAllowedIPForm + + +def create_instance(instance_id): + return WireGuardInstance.objects.create( + instance_id=instance_id, private_key=f'private-{instance_id}', public_key=f'public-{instance_id}', + hostname='primary.example.test', listen_port=51820 + instance_id, address=f'10.188.{instance_id}.1', + ) + + +def create_peer(instance, name, address): + peer = Peer.objects.create( + name=name, public_key=f'public-{name}', pre_shared_key=f'psk-{name}', wireguard_instance=instance, + ) + PeerAllowedIP.objects.create(peer=peer, config_file='server', priority=0, allowed_ip=address, netmask=32) + return peer + + +class NetworkSetupMixin: + """wg0 already routes 192.168.74.0/24 to one of its peers.""" + + def setUp(self): + super().setUp() + self.wg0 = create_instance(0) + self.wg1 = create_instance(1) + self.peer_wg0 = create_peer(self.wg0, 'peer-wg0', '10.188.0.2') + self.peer_wg0_other = create_peer(self.wg0, 'peer-wg0-other', '10.188.0.3') + self.peer_wg1 = create_peer(self.wg1, 'peer-wg1', '10.188.1.2') + self.network_wg0 = PeerAllowedIP.objects.create( + peer=self.peer_wg0, config_file='server', priority=1, allowed_ip='192.168.74.0', netmask=24, + ) + + +class NetworkInOtherInstanceTests(NetworkSetupMixin, TestCase): + """The same network cannot be routed to peers of two instances: the second interface would fail to start.""" + + def form(self, peer, allowed_ip, netmask, config_file='server', instance=None): + return PeerAllowedIPForm( + data={'allowed_ip': allowed_ip, 'netmask': netmask, 'priority': 1}, + instance=instance, current_peer=peer, config_file=config_file, + ) + + def test_form_refuses_network_of_other_instance(self): + form = self.form(self.peer_wg1, '192.168.74.0', 24) + self.assertFalse(form.is_valid()) + self.assertIn('peer-wg0', str(form.errors)) + self.assertIn('wg0', str(form.errors)) + + def test_form_refuses_same_network_written_with_host_address(self): + self.assertFalse(self.form(self.peer_wg1, '192.168.74.9', 24).is_valid()) + + def test_form_accepts_different_prefix(self): + self.assertTrue(self.form(self.peer_wg1, '192.168.74.128', 25).is_valid()) + + def test_form_accepts_same_instance_and_editing_itself(self): + self.assertTrue(self.form(self.peer_wg0_other, '192.168.74.0', 24).is_valid()) + self.assertTrue(self.form(self.peer_wg0, '192.168.74.0', 24, instance=self.network_wg0).is_valid()) + + def test_form_does_not_check_client_routes(self): + self.assertTrue(self.form(self.peer_wg1, '192.168.74.0', 24, config_file='client').is_valid()) + + +class NetworkInOtherInstanceApiTests(NetworkSetupMixin, TestCase): + def setUp(self): + super().setUp() + self.api_key = ApiKey.objects.create(name='tests') + + def call(self, method, payload): + payload = dict(payload, instance='wg1', skip_reload=True) + return getattr(self.client, method)( + '/api/v2/manage_peer/', data=json.dumps(payload), content_type='application/json', + HTTP_TOKEN=str(self.api_key.token), + ) + + def test_api_create_refuses_network_of_other_instance(self): + peer_count = Peer.objects.count() + response = self.call('post', {'name': 'api-peer', 'announced_networks': ['192.168.74.0/24']}) + self.assertEqual(response.status_code, 400) + self.assertIn('wg0', response.json()['error_message']) + self.assertEqual(Peer.objects.count(), peer_count) + + def test_api_update_refuses_network_of_other_instance(self): + response = self.call('put', { + 'peer_public_key': self.peer_wg1.public_key, 'announced_networks': ['192.168.75.0/24', '192.168.74.0/24'], + }) + self.assertEqual(response.status_code, 400) + self.assertFalse(PeerAllowedIP.objects.filter(peer=self.peer_wg1, priority__gte=1).exists()) + + def test_api_update_accepts_free_network(self): + response = self.call('put', {'peer_public_key': self.peer_wg1.public_key, 'announced_networks': ['192.168.75.0/24']}) + self.assertEqual(response.status_code, 200, response.content) + self.assertTrue(PeerAllowedIP.objects.filter(peer=self.peer_wg1, allowed_ip='192.168.75.0', netmask=24).exists()) diff --git a/wireguard_peer/views.py b/wireguard_peer/views.py index 2df1731..5b86cd8 100644 --- a/wireguard_peer/views.py +++ b/wireguard_peer/views.py @@ -6,7 +6,7 @@ from django.http import Http404 from django.shortcuts import get_object_or_404, redirect, render from django.utils.translation import gettext_lazy as _ -from cluster.models import ClusterSettings, Worker +from cluster.models import ClusterSettings from routing_templates.models import RoutingTemplate from scheduler.models import PeerScheduling from user_manager.models import UserAcl @@ -15,7 +15,7 @@ from wgwadmlibrary.tools import check_sort_order_conflict, deduplicate_sort_orde from wireguard.models import Peer, PeerAllowedIP, WireGuardInstance from wireguard_peer.forms import PeerAllowedIPForm, PeerNameForm, PeerKeepaliveForm, PeerKeysForm, PeerSuspensionForm, \ PeerScheduleProfileForm -from wireguard_tools.functions import func_reload_wireguard_interface +from wireguard_tools.functions import func_reload_wireguard_interface, get_config_servers from wireguard_tools.views import export_wireguard_configuration from .functions import func_create_new_peer @@ -24,6 +24,12 @@ def _auto_apply(request, instance): if not settings.AUTO_APPLY: return False export_wireguard_configuration(instance) + hosting_worker = instance.hosting_worker + if hosting_worker: + messages.info(request, _('wg%(id)s runs on %(worker)s, which applies the change on its next synchronization.') % { + 'id': instance.instance_id, 'worker': hosting_worker + }) + return True success, message = func_reload_wireguard_interface(instance) if success: messages.info(request, _('wg%(id)s reloaded automatically.') % {'id': instance.instance_id}) @@ -88,13 +94,7 @@ def view_wireguard_peer_list(request): refresh_interval = settings.WIREGUARD_STATUS_CACHE_REFRESH_INTERVAL cluster_settings = ClusterSettings.objects.filter(name='cluster_settings', enabled=True).first() - servers = [] - if cluster_settings: - if cluster_settings.primary_enable_wireguard: - servers.append({'name': _('Primary Server'), 'uuid': ''}) - - for worker in Worker.objects.filter(enabled=True): - servers.append({'name': worker.display_name, 'uuid': str(worker.uuid)}) + servers = get_config_servers(current_instance) context = { 'page_title': page_title, 'wireguard_instances': wireguard_instances, diff --git a/wireguard_tools/functions.py b/wireguard_tools/functions.py index cf81562..67180c8 100644 --- a/wireguard_tools/functions.py +++ b/wireguard_tools/functions.py @@ -1,11 +1,110 @@ +import ipaddress import os import subprocess from typing import Union +from django.utils.translation import gettext_lazy as _ + +from cluster.models import ClusterSettings, Worker from wireguard.models import WireGuardInstance +WIREGUARD_CONFIG_DIR = '/etc/wireguard' +# Configurations of instances that run on workers (distributed cluster mode). Kept out of WIREGUARD_CONFIG_DIR +# so the primary server does not bring them up. +CLUSTER_WIREGUARD_CONFIG_DIR = '/etc/wireguard/cluster' + + +def get_instance_config_dir(instance: WireGuardInstance) -> str: + return CLUSTER_WIREGUARD_CONFIG_DIR if instance.hosting_worker else WIREGUARD_CONFIG_DIR + + +def get_config_servers(instance: WireGuardInstance) -> list: + """Servers offered when downloading a peer configuration of the instance. Empty without the cluster.""" + cluster_settings = ClusterSettings.objects.filter(name='cluster_settings', enabled=True).first() + if not cluster_settings: + return [] + + if ClusterSettings.is_distributed(): + hosting_worker = instance.hosting_worker if instance else None + if hosting_worker: + return [{'name': hosting_worker.display_name, 'uuid': str(hosting_worker.uuid)}] + return [{'name': _('Primary Server'), 'uuid': ''}] + + servers = [] + if cluster_settings.primary_enable_wireguard: + servers.append({'name': _('Primary Server'), 'uuid': ''}) + for worker in Worker.objects.filter(enabled=True): + servers.append({'name': worker.display_name, 'uuid': str(worker.uuid)}) + return servers + + +def sync_interface_routes(interface_name: str) -> list: + """ + Bring the IPv4 routes of a running interface in line with its allowed IPs, as wg-quick does on start. + wg syncconf only updates the peers, so networks added to or removed from a peer would otherwise only + get their routes on the next restart. Default routes (/0) are left to wg-quick. Returns the errors found. + """ + errors = [] + allowed_ips_result = subprocess.run(['wg', 'show', interface_name, 'allowed-ips'], capture_output=True, text=True) + if allowed_ips_result.returncode != 0: + return [allowed_ips_result.stderr.strip()] + + allowed_networks = set() + for line in allowed_ips_result.stdout.splitlines(): + for value in line.split()[1:]: + try: + network = ipaddress.IPv4Network(value, strict=False) + except ValueError: + continue + if network.prefixlen: + allowed_networks.add(network) + allowed_networks = sorted(allowed_networks, key=lambda network: network.prefixlen) + + for network in allowed_networks: + covering_routes = subprocess.run( + ['ip', '-4', 'route', 'show', 'dev', interface_name, 'match', str(network)], capture_output=True, text=True, + ).stdout.strip() + if covering_routes: + continue + # replace instead of add: the network may still be routed to another instance it was moved from + add_result = subprocess.run( + ['ip', '-4', 'route', 'replace', str(network), 'dev', interface_name], capture_output=True, text=True + ) + if add_result.returncode != 0: + errors.append(f"{network}: {add_result.stderr.strip()}") + + routes_result = subprocess.run( + ['ip', '-o', '-4', 'route', 'show', 'dev', interface_name], capture_output=True, text=True + ) + for route in routes_result.stdout.splitlines(): + fields = route.split() + if not fields or 'kernel' in fields: + continue + try: + route_network = ipaddress.IPv4Network(fields[0], strict=False) + except ValueError: + continue + # Only routes that no allowed IP covers are removed: the interface drops their traffic anyway + if any(route_network.subnet_of(network) for network in allowed_networks): + continue + delete_result = subprocess.run( + ['ip', '-4', 'route', 'delete', str(route_network), 'dev', interface_name], capture_output=True, text=True, + ) + if delete_result.returncode != 0: + errors.append(f"{route_network}: {delete_result.stderr.strip()}") + return errors + def func_reload_wireguard_interface(target: Union[str, WireGuardInstance], config_dir: str = "/etc/wireguard") -> tuple[bool, str]: + success, message = reload_wireguard_interface(target, config_dir) + if not success and isinstance(target, WireGuardInstance): + # Keep the apply banner so the configuration is applied again with a restart + target.pending_changes = True + WireGuardInstance.objects.filter(pk=target.pk).update(pending_changes=True) + return success, message + + +def reload_wireguard_interface(target: Union[str, WireGuardInstance], config_dir: str) -> tuple[bool, str]: """ Reload a WireGuard interface safely using: wg syncconf <(wg-quick strip ) @@ -21,6 +120,9 @@ def func_reload_wireguard_interface(target: Union[str, WireGuardInstance], confi # Resolve interface name and config path if isinstance(target, WireGuardInstance): + if target.hosting_worker: + # The worker applies the new configuration on its next synchronization + return True, f"wg{target.instance_id} runs on {target.hosting_worker} and is updated by the cluster" interface_name = f"wg{target.instance_id}" conf_filename = f"{interface_name}.conf" else: @@ -59,6 +161,10 @@ def func_reload_wireguard_interface(target: Union[str, WireGuardInstance], confi if sync_proc.returncode != 0: return False, sync_proc.stderr.strip() + route_errors = sync_interface_routes(interface_name) + if route_errors: + return False, f"{interface_name} reloaded with route errors: {'; '.join(route_errors)}" + return True, f"{interface_name} reloaded successfully" except subprocess.CalledProcessError as e: diff --git a/wireguard_tools/tests.py b/wireguard_tools/tests.py index 7ce503c..ea212f9 100644 --- a/wireguard_tools/tests.py +++ b/wireguard_tools/tests.py @@ -1,3 +1,258 @@ +import importlib.util +import ipaddress +import os +import subprocess +import tempfile +from unittest.mock import patch + +from django.conf import settings +from django.contrib.auth.models import User +from django.contrib.messages import get_messages from django.test import TestCase -# Create your tests here. +from user_manager.models import UserAcl +from wireguard.models import WireGuardInstance +from wireguard_tools import functions + + +def load_cluster_worker(): + module_path = os.path.join(settings.BASE_DIR, 'containers', 'cluster_node', 'cluster_worker.py') + spec = importlib.util.spec_from_file_location('cluster_worker_under_test', module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class FakeNetwork: + """ + Minimal stand-in for the wg and ip commands used to synchronize routes. Keeps the routes of one interface and + answers 'route show ... match' like iproute2: routes whose prefix contains the given network. + """ + + def __init__(self, interface, allowed_ips, routes): + self.interface = interface + self.allowed_ips = allowed_ips + # {network: is_kernel_route} + self.routes = {ipaddress.ip_network(route): kernel for route, kernel in routes.items()} + self.commands = [] + + def route_lines(self): + return [ + f'{network} proto kernel scope link src x' if kernel else f'{network} scope link ' + for network, kernel in sorted(self.routes.items(), key=lambda item: str(item[0])) + ] + + def run(self, command, *args, **kwargs): + self.commands.append(command) + stdout = '' + if command[:2] == ['wg', 'show']: + stdout = ''.join(f'peer{index}\t{value}\n' for index, value in enumerate(self.allowed_ips)) + elif command[0] == 'ip': + arguments = [value for value in command[1:] if value not in ('-o', '-4')] + action = arguments[1] + if action == 'show' and 'match' in arguments: + network = ipaddress.ip_network(arguments[-1]) + stdout = '\n'.join( + line for line in self.route_lines() + if network.subnet_of(ipaddress.ip_network(line.split()[0])) + ) + elif action == 'show': + stdout = '\n'.join(self.route_lines()) + elif action == 'replace': + self.routes[ipaddress.ip_network(arguments[2])] = False + elif action == 'delete': + self.routes.pop(ipaddress.ip_network(arguments[2]), None) + return subprocess.CompletedProcess(command, 0, stdout, '') + + def route_names(self): + return sorted(str(network) for network in self.routes) + + +class RouteSyncCases: + """Cases shared by the primary server and the cluster node agent implementations.""" + + def sync(self, network): + raise NotImplementedError + + def test_adds_route_of_new_allowed_network(self): + network = FakeNetwork('wg0', ['10.188.0.2/32 192.168.60.0/24'], {'10.188.0.0/24': True}) + self.sync(network) + self.assertEqual(network.route_names(), ['10.188.0.0/24', '192.168.60.0/24']) + + def test_does_not_add_route_already_covered(self): + network = FakeNetwork( + 'wg0', ['10.188.0.2/32 192.168.15.128/25', '10.188.0.3/32 192.168.15.0/24'], {'10.188.0.0/24': True} + ) + self.sync(network) + # Peer addresses are covered by the instance route and the /25 by the /24 added first + self.assertEqual(network.route_names(), ['10.188.0.0/24', '192.168.15.0/24']) + replaced = [command[4] for command in network.commands if command[0] == 'ip' and 'replace' in command] + self.assertEqual(replaced, ['192.168.15.0/24']) + + def test_removes_route_not_covered_by_allowed_ips(self): + network = FakeNetwork( + 'wg0', ['10.188.0.2/32 192.168.15.0/24'], + {'10.188.0.0/24': True, '192.168.15.0/24': False, '192.168.15.128/25': False, '192.168.99.0/24': False}, + ) + self.sync(network) + # The kernel route and the route covered by an allowed network stay + self.assertEqual(network.route_names(), ['10.188.0.0/24', '192.168.15.0/24', '192.168.15.128/25']) + + def test_keeps_kernel_route_without_peers(self): + network = FakeNetwork('wg0', ['(none)'], {'10.188.0.0/24': True, '192.168.60.0/24': False}) + self.sync(network) + self.assertEqual(network.route_names(), ['10.188.0.0/24']) + + def test_ignores_default_route(self): + network = FakeNetwork('wg0', ['10.188.0.2/32 0.0.0.0/0'], {'10.188.0.0/24': True}) + self.sync(network) + self.assertEqual(network.route_names(), ['10.188.0.0/24']) + self.assertFalse([command for command in network.commands if 'replace' in command]) + + +class PrimaryRouteSyncTests(RouteSyncCases, TestCase): + def sync(self, network): + with patch('wireguard_tools.functions.subprocess.run', side_effect=network.run): + self.assertEqual(functions.sync_interface_routes(network.interface), []) + + def test_reload_syncs_routes_after_syncconf(self): + network = FakeNetwork('wg0', ['10.188.0.2/32 192.168.60.0/24'], {'10.188.0.0/24': True}) + with patch('wireguard_tools.functions.os.path.exists', return_value=True), \ + patch('wireguard_tools.functions.os.remove'), \ + patch('builtins.open'), \ + patch('wireguard_tools.functions.subprocess.run', side_effect=network.run): + success, message = functions.func_reload_wireguard_interface('wg0') + self.assertTrue(success, message) + self.assertIn('192.168.60.0/24', network.route_names()) + + +class AgentRouteSyncTests(RouteSyncCases, TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.cluster_worker = load_cluster_worker() + + def sync(self, network): + with patch.object(self.cluster_worker.subprocess, 'run', side_effect=network.run): + self.cluster_worker.sync_interface_routes(network.interface) + + def test_sync_peers_syncs_routes(self): + network = FakeNetwork('wg0', ['10.188.0.2/32 192.168.60.0/24'], {'10.188.0.0/24': True}) + with patch.object(self.cluster_worker.subprocess, 'run', side_effect=network.run): + self.cluster_worker.ClusterNodeAgent.sync_peers('wg0', '/etc/wireguard/wg0.conf') + self.assertIn('192.168.60.0/24', network.route_names()) + + +class RestartInterfacesTests(TestCase): + def setUp(self): + user = User.objects.create_user(username='admin', password='unused') + UserAcl.objects.create(user=user, user_level=50) + self.client.force_login(user) + + def test_restart_does_not_stop_interface_that_is_not_running(self): + commands = [] + + def run(command, *args, **kwargs): + commands.append(command if isinstance(command, str) else ' '.join(command)) + if command == ['wg', 'show', 'interfaces']: + return subprocess.CompletedProcess(command, 0, 'wg0\n', '') + return subprocess.CompletedProcess(command, 0, '', '') + + with patch('wireguard_tools.views.os.listdir', return_value=['wg0.conf', 'wg1.conf']), \ + patch('wireguard_tools.views.os.path.isfile', return_value=True), \ + patch('wireguard_tools.views.subprocess.run', side_effect=run): + response = self.client.get('/tools/restart_wireguard/') + + self.assertIn('wg-quick down wg0', commands) + self.assertNotIn('wg-quick down wg1', commands) + self.assertIn('wg-quick up wg1', commands) + message_texts = [str(message) for message in get_messages(response.wsgi_request)] + self.assertFalse([text for text in message_texts if 'Error' in text], message_texts) + + +class ReloadFailureTests(TestCase): + def setUp(self): + self.instance = WireGuardInstance.objects.create( + instance_id=0, private_key='private-0', public_key='public-0', hostname='primary.example.test', + listen_port=51820, address='10.188.0.1', pending_changes=False, + ) + + def test_failed_reload_marks_instance_pending(self): + with patch('wireguard_tools.functions.reload_wireguard_interface', return_value=(False, 'syncconf failed')): + success, message = functions.func_reload_wireguard_interface(self.instance) + self.assertFalse(success) + self.assertTrue(self.instance.pending_changes) + self.instance.refresh_from_db() + self.assertTrue(self.instance.pending_changes) + + def test_successful_reload_keeps_instance_applied(self): + with patch('wireguard_tools.functions.reload_wireguard_interface', return_value=(True, 'wg0 reloaded')): + success, message = functions.func_reload_wireguard_interface(self.instance) + self.assertTrue(success) + self.instance.refresh_from_db() + self.assertFalse(self.instance.pending_changes) + + def test_route_error_fails_the_reload(self): + network = FakeNetwork('wg0', ['10.188.0.2/32 192.168.60.0/24'], {'10.188.0.0/24': True}) + + def run(command, *args, **kwargs): + if 'replace' in command: + return subprocess.CompletedProcess(command, 2, '', 'RTNETLINK answers: failure') + return network.run(command) + + with patch('wireguard_tools.functions.os.path.exists', return_value=True), \ + patch('wireguard_tools.functions.os.remove'), \ + patch('builtins.open'), \ + patch('wireguard_tools.functions.subprocess.run', side_effect=run): + success, message = functions.func_reload_wireguard_interface(self.instance) + self.assertFalse(success) + self.assertIn('192.168.60.0/24', message) + self.instance.refresh_from_db() + self.assertTrue(self.instance.pending_changes) + + +class AgentInterfaceFailureTests(TestCase): + """A failed interface must be applied again on the next attempt, even if the content did not change.""" + + OLD_CONFIG = '[Interface]\nListenPort = 51820\n\n[Peer]\nPublicKey = peer-a\nAllowedIPs = 10.188.0.2/32\n' + NEW_CONFIG = OLD_CONFIG + 'AllowedIPs = 192.168.60.0/24\n' + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.cluster_worker = load_cluster_worker() + + def setUp(self): + temp_dir = tempfile.TemporaryDirectory() + self.addCleanup(temp_dir.cleanup) + self.config_path = os.path.join(temp_dir.name, 'wg0.conf') + with open(self.config_path, 'w', encoding='utf-8') as config_file: + config_file.write(self.OLD_CONFIG) + for name, value in (('WIREGUARD_DIR', temp_dir.name), ('interface_is_up', lambda interface: True)): + patcher = patch.object(self.cluster_worker, name, value) + patcher.start() + self.addCleanup(patcher.stop) + self.agent = self.cluster_worker.ClusterNodeAgent() + self.section = {'files': {'wg0.conf': self.NEW_CONFIG}} + + def read_config(self): + with open(self.config_path, encoding='utf-8') as config_file: + return config_file.read() + + def test_failed_sync_restores_previous_content_and_retries(self): + SectionError = self.cluster_worker.SectionError + with patch.object(self.cluster_worker.ClusterNodeAgent, 'sync_peers', side_effect=SectionError('wg0 routes: failure')): + with self.assertRaises(SectionError): + self.agent.apply_wireguard(self.section) + self.assertEqual(self.read_config(), self.OLD_CONFIG) + + with patch.object(self.cluster_worker.ClusterNodeAgent, 'sync_peers') as sync_peers: + self.agent.apply_wireguard(self.section) + sync_peers.assert_called_once_with('wg0', self.config_path) + self.assertEqual(self.read_config(), self.NEW_CONFIG) + + def test_unchanged_content_is_not_applied_again(self): + with patch.object(self.cluster_worker.ClusterNodeAgent, 'sync_peers') as sync_peers: + self.agent.apply_wireguard(self.section) + self.agent.apply_wireguard(self.section) + sync_peers.assert_called_once() diff --git a/wireguard_tools/views.py b/wireguard_tools/views.py index 22219bf..062651a 100644 --- a/wireguard_tools/views.py +++ b/wireguard_tools/views.py @@ -15,13 +15,16 @@ from django.shortcuts import get_object_or_404, redirect, render from django.utils import timezone from django.utils.translation import gettext_lazy as _ -from cluster.models import Worker +from cluster.models import ClusterSettings, Worker from firewall.models import RedirectRule, FirewallSettings from firewall.tools import generate_firewall_script from user_manager.models import UserAcl from vpn_invite.models import PeerInvite from wgwadmlibrary.tools import user_has_access_to_peer from wireguard.models import Peer, PeerAllowedIP, WireGuardInstance +from wireguard_tools.functions import ( + CLUSTER_WIREGUARD_CONFIG_DIR, WIREGUARD_CONFIG_DIR, get_instance_config_dir, sync_interface_routes +) def clean_command_field(command_field): @@ -45,7 +48,11 @@ def generate_peer_config(peer_uuid, server_address=None): if dns_line and wg_instance.dns_search_domains: dns_line += f", {wg_instance.dns_search_domains}" - if server_address: + hosting_worker = wg_instance.hosting_worker + if hosting_worker and hosting_worker.server_address: + # Distributed cluster mode: the instance only runs on this worker + endpoint = f"{hosting_worker.server_address}:{wg_instance.listen_port}" + elif server_address: endpoint = server_address else: endpoint = f"{wg_instance.hostname}:{wg_instance.listen_port}" @@ -77,16 +84,19 @@ def export_firewall_configuration(): def set_instance_to_include_firewall(): firewall_settings, firewall_settings_created = FirewallSettings.objects.get_or_create(name='global') - if firewall_settings.instance_to_execute_firewall: - force_export_all_instances = False - instance_to_execute_firewall = firewall_settings.instance_to_execute_firewall - else: - force_export_all_instances = True - instance_to_execute_firewall = WireGuardInstance.objects.order_by('instance_id').first() - if instance_to_execute_firewall: - firewall_settings.instance_to_execute_firewall = instance_to_execute_firewall - firewall_settings.save() - return instance_to_execute_firewall, force_export_all_instances + instance_to_execute_firewall = firewall_settings.instance_to_execute_firewall + # The firewall script runs from the PostUp of an instance that runs on the primary server + if instance_to_execute_firewall and not instance_to_execute_firewall.hosting_worker: + return instance_to_execute_firewall, False + + primary_instances = [ + instance for instance in WireGuardInstance.objects.order_by('instance_id') if not instance.hosting_worker + ] + instance_to_execute_firewall = primary_instances[0] if primary_instances else None + if firewall_settings.instance_to_execute_firewall != instance_to_execute_firewall: + firewall_settings.instance_to_execute_firewall = instance_to_execute_firewall + firewall_settings.save() + return instance_to_execute_firewall, True def export_wireguard_configuration(instance_only: WireGuardInstance = None): @@ -99,24 +109,31 @@ def export_wireguard_configuration(instance_only: WireGuardInstance = None): else: instances = WireGuardInstance.objects.all() - base_dir = "/etc/wireguard" - os.makedirs(base_dir, exist_ok=True) + os.makedirs(WIREGUARD_CONFIG_DIR, exist_ok=True) + os.makedirs(CLUSTER_WIREGUARD_CONFIG_DIR, exist_ok=True) + config_dirs = {instance.uuid: get_instance_config_dir(instance) for instance in instances} firewall_inserted = False - if cleanup_orphaned: - active_instance_conf_set = {f"wg{instance.instance_id}.conf" for instance in instances} - for old_conf_file in glob.glob(os.path.join(base_dir, "wg[0-9]*.conf")): + for config_dir in (WIREGUARD_CONFIG_DIR, CLUSTER_WIREGUARD_CONFIG_DIR): + expected_conf_set = { + f"wg{instance.instance_id}.conf" for instance in instances if config_dirs[instance.uuid] == config_dir + } + if cleanup_orphaned: + orphaned_files = glob.glob(os.path.join(config_dir, "wg[0-9]*.conf")) + else: + # An exported instance may have moved between the primary server and a worker + orphaned_files = [ + os.path.join(config_dir, f"wg{instance.instance_id}.conf") + for instance in instances if config_dirs[instance.uuid] != config_dir + ] + for old_conf_file in orphaned_files: filename = os.path.basename(old_conf_file) - if filename not in active_instance_conf_set: - logging.info("Removing abandoned WireGuard config: %s", filename) + if filename not in expected_conf_set and os.path.isfile(old_conf_file): + logging.info("Removing abandoned WireGuard config: %s", old_conf_file) try: os.remove(old_conf_file) - except FileNotFoundError: - pass - except IsADirectoryError: - continue - except PermissionError: + except (FileNotFoundError, PermissionError): continue for instance in instances: @@ -213,7 +230,7 @@ def export_wireguard_configuration(instance_only: WireGuardInstance = None): config_lines.append("") config_content = "\n".join(config_lines) - config_path = os.path.join(base_dir, f"wg{instance.instance_id}.conf") + config_path = os.path.join(config_dirs[instance.uuid], f"wg{instance.instance_id}.conf") with open(config_path, "w") as config_file: config_file.write(config_content) @@ -264,7 +281,8 @@ def download_config_or_qrcode(request): worker_uuid = request.GET.get('worker') server_address = None - if worker_uuid: + # In the distributed cluster mode the endpoint is always the server that runs the instance + if worker_uuid and not ClusterSettings.is_distributed(): try: worker = Worker.objects.get(uuid=worker_uuid) server_address = f"{worker.server_address}:{peer.wireguard_instance.listen_port}" @@ -307,6 +325,18 @@ def restart_wireguard_interfaces(request): config_dir = "/etc/wireguard" interface_count = 0 error_count = 0 + + # Interfaces without a configuration on this server, such as instances moved to a worker + running_interfaces = subprocess.run(['wg', 'show', 'interfaces'], capture_output=True, text=True).stdout.split() + for interface_name in running_interfaces: + if re.fullmatch(r'wg[0-9]+', interface_name) and not os.path.isfile(os.path.join(config_dir, f'{interface_name}.conf')): + stop_result = subprocess.run(['ip', 'link', 'delete', interface_name], capture_output=True, text=True) + if stop_result.returncode != 0: + messages.warning(request, _("Error stopping") + f" {interface_name}|{stop_result.stderr}") + error_count += 1 + else: + interface_count += 1 + for filename in os.listdir(config_dir): if filename.endswith(".conf"): interface_name = filename[:-5] @@ -332,8 +362,9 @@ def restart_wireguard_interfaces(request): result = subprocess.run(reload_command, shell=True, capture_output=True, text=True) os.remove(temp_config_path) - if result.returncode != 0: - messages.warning(request, _('Error reloading') + f" {interface_name}|{result.stderr}") + reload_error = result.stderr if result.returncode != 0 else '; '.join(sync_interface_routes(interface_name)) + if reload_error: + messages.warning(request, _('Error reloading') + f" {interface_name}|{reload_error}") error_count += 1 else: interface_count += 1 @@ -342,11 +373,13 @@ def restart_wireguard_interfaces(request): if not user_acl.enable_restart: return render(request, 'access_denied.html', {'page_title': 'Access Denied'}) - stop_command = f"wg-quick down {interface_name}" - stop_result = subprocess.run(stop_command, shell=True, capture_output=True, text=True) - if stop_result.returncode != 0: - messages.warning(request, _("Error stopping") + f" {interface_name}|{stop_result.stderr}") - error_count += 1 + # New instances and instances moved back from a worker are not running yet + if interface_name in running_interfaces: + stop_command = f"wg-quick down {interface_name}" + stop_result = subprocess.run(stop_command, shell=True, capture_output=True, text=True) + if stop_result.returncode != 0: + messages.warning(request, _("Error stopping") + f" {interface_name}|{stop_result.stderr}") + error_count += 1 start_command = f"wg-quick up {interface_name}" start_result = subprocess.run(start_command, shell=True, capture_output=True, text=True) if start_result.returncode != 0: diff --git a/wireguard_webadmin/settings.py b/wireguard_webadmin/settings.py index 63e372f..7972c0f 100644 --- a/wireguard_webadmin/settings.py +++ b/wireguard_webadmin/settings.py @@ -176,7 +176,7 @@ DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' WIREGUARD_WEBADMIN_VERSION = 9981 -CLUSTER_WORKER_CURRENT_VERSION = 12 +CLUSTER_WORKER_CURRENT_VERSION = 13 CLUSTER_WORKER_MINIMUM_VERSION = 12 CADDY_ENABLED = os.getenv("CADDY_ENABLED", "false").lower() == "true"