Add distributed cluster mode and sync peer network routes on reload

This commit is contained in:
Eduardo Silva
2026-11-20 20:42:42 -03:00
parent 5f6e2cd61c
commit 0ee6c2b864
25 changed files with 993 additions and 70 deletions
+13 -1
View File
@@ -5,7 +5,8 @@ from django.conf import settings
from django.core.exceptions import ValidationError
from django.utils.translation import gettext_lazy as _
from .models import ClusterSettings, Worker
from firewall.models import FirewallRule
from .models import DISTRIBUTED_MODES, ClusterSettings, Worker
class WorkerForm(forms.ModelForm):
@@ -132,4 +133,15 @@ class ClusterSettingsForm(forms.ModelForm):
if cluster_enabled and not settings.WIREGUARD_STATUS_CACHE_ENABLED:
raise ValidationError(_("Cluster mode requires WireGuard status cache to be enabled."))
if cluster_enabled and cleaned_data.get('cluster_mode') in DISTRIBUTED_MODES:
instance_rules = list(FirewallRule.with_instance_interface().order_by('firewall_chain', 'sort_order'))
if instance_rules:
rule_list = ', '.join(
rule.description or f"{rule.get_firewall_chain_display()} #{rule.sort_order}" for rule in instance_rules
)
raise ValidationError(_(
"The distributed cluster mode does not allow firewall rules bound to a specific WireGuard instance. "
"Change them to wg+ or IP addresses first: %(rules)s"
) % {'rules': rule_list})
return cleaned_data
@@ -0,0 +1,18 @@
# Generated by Django 5.2.19 on 2026-11-20 19:13
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('cluster', '0018_cluster_v2'),
]
operations = [
migrations.AlterField(
model_name='clustersettings',
name='cluster_mode',
field=models.CharField(choices=[('mirror', 'Mirror'), ('distributed', 'Distributed')], default='mirror', max_length=16),
),
]
+10 -1
View File
@@ -3,12 +3,16 @@ import uuid
from django.db import models
from django.utils.translation import gettext_lazy as _
DISTRIBUTED_MODES = ('distributed',)
class ClusterSettings(models.Model):
name = models.CharField(default='cluster_settings', max_length=16, unique=True)
enabled = models.BooleanField(default=False)
primary_enable_wireguard = models.BooleanField(default=True)
cluster_mode = models.CharField(default='mirror', max_length=16, choices=(('mirror', 'Mirror'), ))
cluster_mode = models.CharField(
default='mirror', max_length=16, choices=(('mirror', 'Mirror'), ('distributed', 'Distributed'))
)
worker_display = models.CharField(
default='server_address', max_length=16, choices=(
('name', 'Name'), ('server_address', 'Server Address'),
@@ -27,6 +31,11 @@ class ClusterSettings(models.Model):
def cluster_enabled(cls):
return cls.objects.filter(name='cluster_settings', enabled=True).exists()
@classmethod
def is_distributed(cls):
"""Each WireGuard instance runs on a single server. Only applies while the cluster is enabled."""
return cls.objects.filter(name='cluster_settings', enabled=True, cluster_mode__in=DISTRIBUTED_MODES).exists()
class Worker(models.Model):
name = models.CharField(max_length=100, unique=True)
+15 -4
View File
@@ -11,12 +11,13 @@ from django.utils.dateparse import parse_datetime
from app_gateway.caddy_config_export import (
add_certificate_files, certificate_file_name, load_worker_export, read_exported_certificate
)
from cluster.models import WorkerStatus
from cluster.models import ClusterSettings, WorkerStatus
from dns.functions import generate_dnsmasq_config
from dns.models import DNSFilterList
from firewall.tools import generate_firewall_script
from wireguard.models import WireGuardInstance
from wireguard_tools.functions import CLUSTER_WIREGUARD_CONFIG_DIR, WIREGUARD_CONFIG_DIR
WIREGUARD_CONFIG_DIR = '/etc/wireguard'
DNSMASQ_CONFIG_DIR = '/etc/dnsmasq'
DNSMASQ_MAIN_CONFIG = 'wireguard_webadmin_dns.conf'
@@ -67,7 +68,17 @@ class Section:
def build_wireguard_section(worker):
section = Section()
for config_path in sorted(glob.glob(os.path.join(WIREGUARD_CONFIG_DIR, 'wg[0-9]*.conf'))):
if ClusterSettings.is_distributed():
# Only the instances that run on this worker
config_paths = [
os.path.join(CLUSTER_WIREGUARD_CONFIG_DIR, f'wg{instance_id}.conf')
for instance_id in WireGuardInstance.objects.filter(worker=worker).values_list('instance_id', flat=True)
]
else:
config_paths = glob.glob(os.path.join(WIREGUARD_CONFIG_DIR, 'wg[0-9]*.conf'))
for config_path in sorted(config_paths):
if not os.path.isfile(config_path):
continue
content, content_sha256 = _read_file(config_path)
section.add_file(os.path.basename(config_path), content, content_sha256)
return section
@@ -75,7 +86,7 @@ def build_wireguard_section(worker):
def build_firewall_section(worker):
section = Section()
section.add_file('wg-firewall.sh', generate_firewall_script(cluster_node=True))
section.add_file('wg-firewall.sh', generate_firewall_script(cluster_node=True, worker=worker))
return section
+164
View File
@@ -3,6 +3,7 @@ import os
import tempfile
from unittest.mock import patch
from django.contrib.auth.models import User
from django.test import TestCase, override_settings
from django.utils import timezone
@@ -12,9 +13,16 @@ from app_gateway import caddy_config_export
from app_gateway.caddy_config_export import certificate_file_name, export_caddy_config
from app_gateway.models import Application, ApplicationHost, DnsChallengeCredential
from cluster import sync
from cluster.forms import ClusterSettingsForm
from cluster.models import ClusterSettings, Worker, WorkerStatus
from dns.models import DNSSettings
from firewall.forms import FirewallRuleForm
from firewall.models import FirewallRule, FirewallSettings, RedirectRule
from firewall.tools import generate_firewall_script
from user_manager.models import UserAcl
from wireguard.models import Peer, PeerAllowedIP, WireGuardInstance
from wireguard_tools.functions import get_config_servers
from wireguard_tools.views import export_wireguard_configuration, generate_peer_config
FAKE_CERTIFICATE = '-----BEGIN CERTIFICATE-----\nfake\n-----END CERTIFICATE-----\n'
FAKE_PRIVATE_KEY = '-----BEGIN PRIVATE KEY-----\nfake\n-----END PRIVATE KEY-----\n'
@@ -271,3 +279,159 @@ class ClusterTrafficAggregationTests(TestCase):
self.assertEqual(combined['PEER']['endpoints'], 'node:1')
self.assertEqual(combined['PEER']['source_name'], 'worker_node-a')
self.assertEqual(combined['NODE_ONLY']['transfer'], {'tx': 7, 'rx': 3})
def create_instance(instance_id, worker=None):
return WireGuardInstance.objects.create(
instance_id=instance_id, private_key=f'private-{instance_id}', public_key=f'public-{instance_id}',
hostname='primary.example.test', listen_port=51820 + instance_id, address=f'10.188.{instance_id}.1',
worker=worker,
)
@override_settings(WIREGUARD_STATUS_CACHE_ENABLED=True)
class DistributedModeTests(ClusterTestMixin, TestCase):
"""wg0 runs on the primary server, wg1 on node-a and node-b runs no instance."""
def setUp(self):
super().setUp()
self.cluster_wireguard_dir = os.path.join(self.wireguard_dir, 'cluster')
for target, value in (
('cluster.sync.CLUSTER_WIREGUARD_CONFIG_DIR', self.cluster_wireguard_dir),
('wireguard_tools.functions.WIREGUARD_CONFIG_DIR', self.wireguard_dir),
('wireguard_tools.functions.CLUSTER_WIREGUARD_CONFIG_DIR', self.cluster_wireguard_dir),
('wireguard_tools.views.WIREGUARD_CONFIG_DIR', self.wireguard_dir),
('wireguard_tools.views.CLUSTER_WIREGUARD_CONFIG_DIR', self.cluster_wireguard_dir),
):
patcher = patch(target, value)
patcher.start()
self.addCleanup(patcher.stop)
self.worker_b = Worker.objects.create(name='node-b', hostname='node-b.example.test')
self.primary_instance = create_instance(0)
self.worker_instance = create_instance(1, worker=self.worker)
ClusterSettings.objects.filter(name='cluster_settings').update(cluster_mode='distributed')
def set_mode(self, enabled=True, cluster_mode='distributed'):
ClusterSettings.objects.filter(name='cluster_settings').update(enabled=enabled, cluster_mode=cluster_mode)
def create_peer(self, instance):
peer = Peer.objects.create(
name=f'peer-wg{instance.instance_id}', public_key=f'peer-public-{instance.instance_id}',
pre_shared_key='', private_key=f'peer-private-{instance.instance_id}', wireguard_instance=instance,
)
PeerAllowedIP.objects.create(
peer=peer, priority=0, allowed_ip=f'10.188.{instance.instance_id}.10', netmask=32, config_file='server'
)
return peer
def test_hosting_worker_only_in_distributed_mode(self):
self.assertTrue(ClusterSettings.is_distributed())
self.assertEqual(self.worker_instance.hosting_worker, self.worker)
self.assertIsNone(self.primary_instance.hosting_worker)
self.set_mode(cluster_mode='mirror')
self.assertIsNone(WireGuardInstance.objects.get(instance_id=1).hosting_worker)
self.set_mode(enabled=False)
self.assertFalse(ClusterSettings.is_distributed())
self.assertIsNone(WireGuardInstance.objects.get(instance_id=1).hosting_worker)
def test_export_places_configs_by_server(self):
FirewallSettings.objects.update_or_create(name='global', defaults={'instance_to_execute_firewall': self.worker_instance})
export_wireguard_configuration()
self.assertEqual(sorted(os.listdir(self.cluster_wireguard_dir)), ['wg1.conf'])
self.assertIn('wg0.conf', os.listdir(self.wireguard_dir))
self.assertNotIn('wg1.conf', os.listdir(self.wireguard_dir))
# The firewall script must run from an instance of the primary server
self.assertEqual(FirewallSettings.objects.get(name='global').instance_to_execute_firewall, self.primary_instance)
with open(os.path.join(self.wireguard_dir, 'wg0.conf'), encoding='utf-8') as config_file:
self.assertIn('PostUp = /etc/wireguard/wg-firewall.sh', config_file.read())
# Moving a single instance back to the primary server removes it from the cluster directory
WireGuardInstance.objects.filter(instance_id=1).update(worker=None)
export_wireguard_configuration(WireGuardInstance.objects.get(instance_id=1))
self.assertIn('wg1.conf', os.listdir(self.wireguard_dir))
self.assertEqual(os.listdir(self.cluster_wireguard_dir), [])
def test_wireguard_section_only_has_worker_instances(self):
export_wireguard_configuration()
self.assertEqual(sorted(sync.build_section('wireguard', self.worker).files), ['wg1.conf'])
self.assertEqual(sync.build_section('wireguard', self.worker_b).files, {})
self.set_mode(cluster_mode='mirror')
export_wireguard_configuration()
self.assertEqual(sorted(sync.build_section('wireguard', self.worker_b).files), ['wg0.conf', 'wg1.conf'])
def test_port_forwards_only_on_instance_server(self):
RedirectRule.objects.create(port=8081, wireguard_instance=self.worker_instance, ip_address='10.188.1.10')
RedirectRule.objects.create(port=8082, wireguard_instance=self.primary_instance, ip_address='10.188.0.10')
worker_script = generate_firewall_script(cluster_node=True, worker=self.worker)
other_worker_script = generate_firewall_script(cluster_node=True, worker=self.worker_b)
primary_script = generate_firewall_script()
self.assertIn('--dport 8081', worker_script)
self.assertNotIn('--dport 8082', worker_script)
self.assertNotIn('--dport 808', other_worker_script)
self.assertIn('--dport 8082', primary_script)
self.assertNotIn('--dport 8081', primary_script)
self.set_mode(cluster_mode='mirror')
self.assertIn('--dport 8082', generate_firewall_script(cluster_node=True, worker=self.worker_b))
def test_activation_blocked_by_instance_interface_rules(self):
self.set_mode(cluster_mode='mirror')
FirewallRule.objects.create(description='wg0 to internet', in_interface='wg0', out_interface='eth0')
cluster_settings = ClusterSettings.objects.get(name='cluster_settings')
data = {'enabled': True, 'primary_enable_wireguard': True, 'cluster_mode': 'distributed', 'worker_display': 'name'}
form = ClusterSettingsForm(data=data, instance=cluster_settings)
self.assertFalse(form.is_valid())
self.assertIn('wg0 to internet', str(form.non_field_errors()))
FirewallRule.objects.update(in_interface='wg+')
self.assertTrue(ClusterSettingsForm(data=data, instance=cluster_settings).is_valid())
def test_firewall_rule_form_rejects_instance_interface(self):
def rule_form(in_interface):
return FirewallRuleForm(data={
'firewall_chain': 'forward', 'in_interface': in_interface, 'out_interface': '', 'rule_action': 'accept',
'source_netmask': 32, 'destination_netmask': 32, 'protocol': '', 'sort_order': 10,
})
form = rule_form('wg0')
self.assertFalse(form.is_valid())
self.assertIn('distributed cluster mode', str(form.non_field_errors()))
self.assertTrue(rule_form('wg+').is_valid())
self.set_mode(cluster_mode='mirror')
self.assertTrue(rule_form('wg0').is_valid())
def test_peer_config_uses_instance_server(self):
worker_peer = self.create_peer(self.worker_instance)
primary_peer = self.create_peer(self.primary_instance)
self.assertIn('Endpoint = node-a.example.test:51821', generate_peer_config(worker_peer.uuid))
# A server picked for another worker is ignored: the instance only runs on node-a
self.assertIn(
'Endpoint = node-a.example.test:51821',
generate_peer_config(worker_peer.uuid, server_address='node-b.example.test:51821'),
)
self.assertIn('Endpoint = primary.example.test:51820', generate_peer_config(primary_peer.uuid))
self.assertEqual(get_config_servers(self.worker_instance), [{'name': 'node-a.example.test', 'uuid': str(self.worker.uuid)}])
self.assertEqual([server['uuid'] for server in get_config_servers(self.primary_instance)], [''])
self.set_mode(cluster_mode='mirror')
self.assertEqual(len(get_config_servers(self.worker_instance)), 3)
def test_worker_with_instances_cannot_be_deleted(self):
user = User.objects.create_user(username='cluster-admin', password='unused-test-password')
UserAcl.objects.create(user=user, user_level=50)
self.client.force_login(user)
response = self.client.get(f'/cluster/worker/manage/?uuid={self.worker.uuid}&action=delete&confirmation=delete')
self.assertRedirects(response, f'/cluster/worker/view/?uuid={self.worker.uuid}', fetch_redirect_response=False)
self.assertTrue(Worker.objects.filter(pk=self.worker.pk).exists())
self.client.get(f'/cluster/worker/manage/?uuid={self.worker_b.uuid}&action=delete&confirmation=delete')
self.assertFalse(Worker.objects.filter(pk=self.worker_b.pk).exists())
+31 -3
View File
@@ -1,6 +1,7 @@
from django.conf import settings
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.db.models import ProtectedError
from django.shortcuts import get_object_or_404, redirect, render
from django.urls import reverse
from django.utils.dateparse import parse_datetime
@@ -9,6 +10,8 @@ from django.utils.translation import gettext_lazy as _
from api.models import WireguardStatusCache
from app_gateway.setup_defaults import get_webadmin_hostnames
from user_manager.models import UserAcl
from wireguard.models import WireGuardInstance
from wireguard_tools.views import export_firewall_configuration, export_wireguard_configuration
from .forms import WorkerForm, ClusterSettingsForm
from .models import ClusterSettings, Worker, WorkerStatus
from .sync import SECTION_NAMES, get_worker_sync_state, get_worker_warnings
@@ -39,6 +42,10 @@ def describe_worker(worker):
worker.services = sorted((worker_status.services_status or {}).items()) if worker_status else []
worker.runs_gateway = any(service == 'caddy' for service, state in worker.services)
worker.outdated = bool(worker_status and worker_status.worker_version != settings.CLUSTER_WORKER_CURRENT_VERSION)
# WireGuard instances that run only on this worker (distributed cluster mode)
worker.hosted_instances = (
list(worker.wireguard_instances.order_by('instance_id')) if ClusterSettings.is_distributed() else []
)
return worker
@@ -63,6 +70,12 @@ def cluster_main(request):
'workers': workers,
'cluster_settings': cluster_settings,
'current_worker_version': settings.CLUSTER_WORKER_CURRENT_VERSION,
'distributed': ClusterSettings.is_distributed(),
'primary_instances': WireGuardInstance.objects.filter(worker__isnull=True).order_by('instance_id'),
# Instances that no server runs: their worker is disabled
'stopped_instances': [
instance for worker in workers if not worker.enabled for instance in worker.hosted_instances
],
'summary': {
'enabled': len(enabled_workers),
'online': sum(1 for worker in enabled_workers if worker.operational),
@@ -119,6 +132,7 @@ def worker_details(request):
'worker': worker,
'cluster_settings': cluster_settings,
'current_worker_version': settings.CLUSTER_WORKER_CURRENT_VERSION,
'distributed': ClusterSettings.is_distributed(),
'sections': sections,
'certificates': certificates,
'primary_address': primary_address,
@@ -140,8 +154,14 @@ def worker_manage(request):
if request.GET.get('action') == 'delete':
worker_name = worker.name
if request.GET.get('confirmation') == 'delete':
worker.delete()
messages.success(request, _('Worker deleted|Worker deleted: ') + worker_name)
try:
worker.delete()
messages.success(request, _('Worker deleted|Worker deleted: ') + worker_name)
except ProtectedError:
messages.warning(request, _(
'Worker not deleted|Move the WireGuard instances of this worker to another server first: '
) + ', '.join(str(instance) for instance in worker.wireguard_instances.all()))
return redirect(reverse('worker_details') + f'?uuid={worker.uuid}')
else:
messages.warning(request, _('Worker not deleted|Invalid confirmation.'))
return redirect(reverse('cluster_main'))
@@ -195,10 +215,17 @@ def cluster_settings(request):
cluster_settings, cluster_settings_created = ClusterSettings.objects.get_or_create(name='cluster_settings')
was_distributed = ClusterSettings.is_distributed()
form = ClusterSettingsForm(request.POST or None, instance=cluster_settings)
if form.is_valid():
form.save()
cluster_settings = form.save()
messages.success(request, _('Cluster settings updated successfully.'))
if ClusterSettings.is_distributed() != was_distributed:
# Where each instance runs may have changed: workers synchronize the new files right away and the
# primary server moves its interfaces when the configuration is applied
export_wireguard_configuration()
export_firewall_configuration()
WireGuardInstance.objects.update(pending_changes=True)
return redirect(reverse('cluster_main'))
form_description = {
@@ -212,6 +239,7 @@ def cluster_settings(request):
<h5>Cluster Mode</h5>
<p><strong>Mirror</strong>: every worker receives the same WireGuard instances, peers, firewall and DNS configuration. Peers can connect to any server.</p>
<p><strong>Distributed</strong>: each WireGuard instance runs only on the server selected in the instance settings (the primary server by default). Peers connect to that server. Firewall rules cannot be bound to a specific instance interface: use wg+ or IP addresses. After changing the mode, apply the WireGuard configuration so the primary server stops the instances that moved to workers.</p>
<h5>Worker Display</h5>
<p>How workers are named in server selections (peer configuration downloads and VPN invites).</p>