Rework cluster into v2: section-based worker sync and App Gateway on workers

This commit is contained in:
Eduardo Silva
2026-11-20 11:56:30 -03:00
parent dee270695d
commit 7468cdf623
41 changed files with 2766 additions and 1014 deletions
+10
View File
@@ -401,11 +401,21 @@ def func_concatenate_cluster_wireguard_status_cache() -> None:
existing_peer = combined_data[interface][peer_key]
# Each server keeps its own counters, so the peer traffic is the sum of all sources
existing_transfer = existing_peer.get("transfer") or {}
new_transfer = peer_info_copy.get("transfer") or {}
combined_transfer = {
"tx": int(existing_transfer.get("tx", 0) or 0) + int(new_transfer.get("tx", 0) or 0),
"rx": int(existing_transfer.get("rx", 0) or 0) + int(new_transfer.get("rx", 0) or 0),
}
new_hs = _latest_handshake_as_int(peer_info_copy)
old_hs = _latest_handshake_as_int(existing_peer)
# Handshake, endpoint and source come from the server where the peer connected last
if new_hs > old_hs:
combined_data[interface][peer_key] = peer_info_copy
combined_data[interface][peer_key]["transfer"] = combined_transfer
processing_time_ms = int((time.monotonic() - start_time) * 1000)
WireguardStatusCache.objects.create(data=combined_data, processing_time_ms=processing_time_ms, cache_type="cluster")
+155 -27
View File
@@ -1,11 +1,16 @@
import hashlib
import json
import os
import shutil
from django.conf import settings
from django.contrib import messages
from django.utils.translation import gettext as _
from app_gateway.models import (
AccessPolicy, Application, ApplicationHost, ApplicationPolicy, RESERVED_APP_NAME
)
from cluster.models import Worker
from gatekeeper.models import (
AuthMethod, GatekeeperGroup, GatekeeperIPAddress, GatekeeperUser
)
@@ -16,11 +21,43 @@ POLICY_TYPE_MAP = {
'deny': 'deny',
}
# Written by the Caddy container of the primary server, read-only for Django
CERTIFICATE_EXPORT_DIR = '/caddy_cert_export'
ACME_CERTIFICATE_TYPES = ('http_challenge', 'dns_challenge')
def build_applications_data():
applications = Application.objects.exclude(name=RESERVED_APP_NAME).prefetch_related('hosts')
def get_served_applications(worker=None):
"""Applications published by the primary server (worker=None) or by a cluster worker."""
applications = Application.objects.prefetch_related('hosts', 'workers')
if worker is not None:
applications = applications.exclude(name=RESERVED_APP_NAME)
return [application for application in applications if application.is_served_by(worker)]
def certificate_file_name(certificate):
return hashlib.sha256(certificate.encode()).hexdigest()
def read_exported_certificate(hostname):
"""Return (certificate, private_key) exported for a hostname, or (None, None)."""
if '/' in hostname or hostname in ('', '.', '..'):
return None, None
certificate_dir = os.path.join(CERTIFICATE_EXPORT_DIR, hostname)
try:
with open(os.path.join(certificate_dir, 'certificate.crt'), 'r', encoding='utf-8') as certificate_file:
certificate = certificate_file.read()
with open(os.path.join(certificate_dir, 'certificate.key'), 'r', encoding='utf-8') as key_file:
private_key = key_file.read()
except OSError:
return None, None
return certificate, private_key
def build_applications_data(worker=None):
entries = []
for app in applications:
for app in get_served_applications(worker):
if app.name == RESERVED_APP_NAME:
continue
entry = {
'id': app.name,
'name': app.display_name or app.name,
@@ -67,7 +104,8 @@ def _build_auth_method_entry(method):
return entry
def build_auth_policies_data():
def build_auth_policies_data(worker=None):
# Policies, methods, groups and users are referenced by name, so every server receives all of them
auth_methods = {}
for method in AuthMethod.objects.all():
auth_methods[method.name] = _build_auth_method_entry(method)
@@ -102,13 +140,9 @@ def build_auth_policies_data():
}
def build_routes_data():
applications = (
Application.objects
.prefetch_related('routes__policy')
)
def build_routes_data(worker=None):
entries = {}
for app in applications:
for app in get_served_applications(worker):
try:
app_policy = ApplicationPolicy.objects.get(application=app)
default_policy = app_policy.default_policy.name
@@ -132,15 +166,41 @@ def build_routes_data():
return {'entries': entries}
def build_tls_data():
def build_tls_data(worker=None):
"""
On the primary server every ACME certificate is issued, including the ones of applications that are only
served by cluster workers (certificate_only), and certificates used by workers are exported. Workers never
issue ACME certificates: they receive the exported certificate (certificate_file) or use the internal CA.
"""
hosts = {}
dns_credentials = {}
for application_host in ApplicationHost.objects.select_related('dns_credential'):
served_application_ids = {application.pk for application in get_served_applications(worker)}
application_hosts = ApplicationHost.objects.select_related('dns_credential', 'application').prefetch_related(
'application__workers'
)
for application_host in application_hosts:
application = application_host.application
hostname = application_host.hostname.lower()
is_acme = application_host.certificate_type in ACME_CERTIFICATE_TYPES
host_entry = {
'certificate_type': application_host.certificate_type,
'certificate_fallback': application_host.certificate_fallback,
'certificate_generation': application_host.certificate_generation,
}
if worker is not None:
if application.pk in served_application_ids:
hosts[hostname] = host_entry
continue
served_by_workers = application.name != RESERVED_APP_NAME and application.served_by_workers
if application.pk not in served_application_ids and not (served_by_workers and is_acme):
continue
host_entry['certificate_generation'] = application_host.certificate_generation
if served_by_workers and is_acme:
host_entry['export_certificate'] = True
if application.pk not in served_application_ids:
host_entry['certificate_only'] = True
dns_credential = application_host.dns_credential
if application_host.certificate_type == 'dns_challenge' and dns_credential:
host_entry['dns_credential'] = dns_credential.name
@@ -148,7 +208,7 @@ def build_tls_data():
'provider': dns_credential.provider,
'credentials': dns_credential.credentials,
}
hosts[application_host.hostname.lower()] = host_entry
hosts[hostname] = host_entry
return {
'hosts': hosts,
@@ -166,30 +226,98 @@ EXPORT_FILES = {
}
def export_caddy_config(output_dir):
os.makedirs(output_dir, exist_ok=True)
# Cluster workers follow the configuration exported for them, so changes are only published by an export
WORKER_EXPORT_DIR_NAME = 'cluster'
def get_worker_export_dir(output_dir, worker):
return os.path.join(output_dir, WORKER_EXPORT_DIR_NAME, str(worker.uuid))
def _write_export_files(target_dir, worker=None):
os.makedirs(target_dir, exist_ok=True)
for filename, builder in EXPORT_FILES.items():
filepath = os.path.join(output_dir, filename)
filepath = os.path.join(target_dir, filename)
with open(filepath, 'w', encoding='utf-8') as output_file:
json.dump(builder(), output_file, indent=2)
json.dump(builder(worker), output_file, indent=2)
output_file.write('\n')
def has_pending_export(output_dir):
"""Return True when the current configuration differs from the files last exported to output_dir."""
for filename, builder in EXPORT_FILES.items():
filepath = os.path.join(output_dir, filename)
def export_caddy_config(output_dir):
os.makedirs(output_dir, exist_ok=True)
workers = list(Worker.objects.all())
for worker in workers:
_write_export_files(get_worker_export_dir(output_dir, worker), worker)
workers_export_dir = os.path.join(output_dir, WORKER_EXPORT_DIR_NAME)
if os.path.isdir(workers_export_dir):
worker_uuids = {str(worker.uuid) for worker in workers}
for worker_dir_name in os.listdir(workers_export_dir):
if worker_dir_name not in worker_uuids:
shutil.rmtree(os.path.join(workers_export_dir, worker_dir_name), ignore_errors=True)
# The primary server is written last: its files are the ones Caddy and the auth-gateway watch
_write_export_files(output_dir)
def load_exported_files(target_dir):
"""Return {filename: data} from an export directory, or None when it is missing or incomplete."""
exported_files = {}
for filename in EXPORT_FILES:
try:
with open(filepath, 'r', encoding='utf-8') as exported_file:
exported_data = json.load(exported_file)
with open(os.path.join(target_dir, filename), 'r', encoding='utf-8') as exported_file:
exported_files[filename] = json.load(exported_file)
except (OSError, ValueError):
return True
# Round-trip through JSON so the comparison matches what export_caddy_config writes
if json.loads(json.dumps(builder())) != exported_data:
return None
return exported_files
def load_worker_export(worker, output_dir=None):
return load_exported_files(get_worker_export_dir(output_dir or CADDY_EXPORT_DIR, worker))
def add_certificate_files(tls_data):
"""Point the ACME hosts of a worker to the certificates currently exported by the primary server."""
for hostname, host_tls in tls_data.get('hosts', {}).items():
if host_tls.get('certificate_type') not in ACME_CERTIFICATE_TYPES:
continue
certificate, private_key = read_exported_certificate(hostname)
host_tls['certificate_file'] = certificate_file_name(certificate) if certificate and private_key else ''
return tls_data
def _export_differs(target_dir, worker=None):
exported_files = load_exported_files(target_dir)
if exported_files is None:
return True
# Round-trip through JSON so the comparison matches what export_caddy_config writes
for filename, builder in EXPORT_FILES.items():
if json.loads(json.dumps(builder(worker))) != exported_files[filename]:
return True
return False
def has_pending_export(output_dir):
"""Return True when the current configuration differs from the files last exported to output_dir."""
if _export_differs(output_dir):
return True
return any(_export_differs(get_worker_export_dir(output_dir, worker), worker) for worker in Worker.objects.all())
def caddy_export_pending():
return settings.CADDY_ENABLED and has_pending_export(CADDY_EXPORT_DIR)
def apply_caddy_config():
"""Export the configuration read by Caddy, the auth-gateway and the cluster workers."""
export_caddy_config(CADDY_EXPORT_DIR)
if settings.DEBUG:
export_caddy_config(os.path.join(settings.BASE_DIR, 'containers', 'caddy', 'config_files'))
def auto_apply_caddy_config(request):
if not settings.AUTO_APPLY or not settings.CADDY_ENABLED:
return False
apply_caddy_config()
messages.info(request, _('Gateway configuration applied automatically.'))
return True
+47 -1
View File
@@ -12,22 +12,44 @@ from app_gateway.models import (
Application, ApplicationHost, AccessPolicy, ApplicationPolicy, ApplicationRoute, DnsChallengeCredential
)
from app_gateway.setup_defaults import is_ip_address
from cluster.models import ClusterSettings, Worker
class ApplicationForm(forms.ModelForm):
class Meta:
model = Application
fields = ['display_name', 'upstream', 'allow_invalid_cert']
fields = ['display_name', 'upstream', 'allow_invalid_cert', 'serve_on_primary', 'all_workers', 'workers']
labels = {
'display_name': _('Name'),
'upstream': _('Upstream'),
'allow_invalid_cert': _('Allow invalid/self-signed certificate'),
'serve_on_primary': _('Serve on the primary server'),
'all_workers': _('Serve on all cluster workers'),
'workers': _('Cluster workers'),
}
def __init__(self, *args, **kwargs):
cancel_url = kwargs.pop('cancel_url', '#')
super().__init__(*args, **kwargs)
self.fields['display_name'].required = True
self.cluster_enabled = ClusterSettings.cluster_enabled()
cluster_rows = []
if self.cluster_enabled:
self.fields['workers'].queryset = Worker.objects.order_by('name')
self.fields['workers'].help_text = _('Select specific workers, or leave empty when serving on all cluster workers.')
cluster_rows = [
HTML(f'<hr><h5>{_("Cluster")}</h5><p class="text-muted">{_("Servers that publish this application.")}</p>'),
Div(
Div('serve_on_primary', css_class='col-md-12'),
Div('all_workers', css_class='col-md-12'),
Div('workers', css_class='col-md-12'),
css_class='row'
),
]
else:
for field_name in ('serve_on_primary', 'all_workers', 'workers'):
del self.fields[field_name]
self.helper = FormHelper()
self.helper.layout = Layout(
@@ -40,6 +62,7 @@ class ApplicationForm(forms.ModelForm):
Div('allow_invalid_cert', css_class='col-md-12'),
css_class='row'
),
*cluster_rows,
Div(
Div(
Submit('submit', _('Save'), css_class='btn btn-primary'),
@@ -71,6 +94,22 @@ class ApplicationForm(forms.ModelForm):
elif parsed.scheme == "http" and cleaned_data.get("allow_invalid_cert"):
self.add_error("allow_invalid_cert", _("This option is only available for https:// upstreams."))
if self.cluster_enabled:
all_workers = cleaned_data.get('all_workers')
selected_workers = cleaned_data.get('workers')
if all_workers and selected_workers:
self.add_error('workers', _('Leave this empty when the application is served on all cluster workers.'))
if not cleaned_data.get('serve_on_primary') and not all_workers and not selected_workers:
self.add_error(None, _('Select at least one server to publish this application.'))
served_by_workers = all_workers or bool(selected_workers)
if served_by_workers and self.instance.pk:
http_hosts = self.instance.hosts.filter(certificate_type='http_challenge').values_list('hostname', flat=True)
if http_hosts:
self.add_error(None, _(
'Applications served by cluster workers require DNS validation or self-signed certificates. '
'Change the certificate of these hosts first: %(hosts)s'
) % {'hosts': ', '.join(http_hosts)})
return cleaned_data
@@ -99,6 +138,12 @@ class ApplicationHostForm(forms.ModelForm):
if hostname.lower() in webadmin_hostnames:
self.add_error('hostname', _('This hostname is already used by WireGuard WebAdmin.'))
application = self.instance.application if self.instance.pk else self.application
if certificate_type == 'http_challenge' and application and application.served_by_workers:
self.add_error('certificate_type', _(
'This application is served by cluster workers. Use DNS validation or a self-signed certificate.'
))
if hostname and is_ip_address(hostname) and certificate_type != 'self_signed':
self.add_error('certificate_type', _('IP addresses can only use a self-signed certificate.'))
@@ -117,6 +162,7 @@ class ApplicationHostForm(forms.ModelForm):
def __init__(self, *args, **kwargs):
cancel_url = kwargs.pop('cancel_url', '#')
self.is_reserved = kwargs.pop('is_reserved', False)
self.application = kwargs.pop('application', None)
super().__init__(*args, **kwargs)
self.fields['dns_credential'].required = False
if self.is_reserved:
@@ -1,11 +1,10 @@
import os
import sys
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from django.db import transaction
from app_gateway.caddy_config_export import export_caddy_config
from app_gateway.caddy_config_export import apply_caddy_config
from app_gateway.models import AccessPolicy, Application, ApplicationPolicy, RESERVED_APP_NAME
from app_gateway.setup_defaults import create_default_entries
@@ -86,9 +85,7 @@ class Command(BaseCommand):
if settings.CADDY_ENABLED:
self.stdout.write('Exporting Caddy configuration...')
export_caddy_config('/caddy_json_export/')
if settings.DEBUG:
export_caddy_config(os.path.join(settings.BASE_DIR, 'containers', 'caddy', 'config_files'))
apply_caddy_config()
self.stdout.write(self.style.SUCCESS('Caddy configuration exported.'))
else:
self.stdout.write('Skipping Caddy configuration export (CADDY_ENABLED is not set).')
@@ -0,0 +1,29 @@
# Generated by Django 5.2.19 on 2026-11-19 16:17
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('app_gateway', '0012_certificate_type_labels'),
('cluster', '0018_cluster_v2'),
]
operations = [
migrations.AddField(
model_name='application',
name='all_workers',
field=models.BooleanField(default=False),
),
migrations.AddField(
model_name='application',
name='serve_on_primary',
field=models.BooleanField(default=True),
),
migrations.AddField(
model_name='application',
name='workers',
field=models.ManyToManyField(blank=True, related_name='applications', to='cluster.worker'),
),
]
+16
View File
@@ -4,6 +4,7 @@ from django.db import models
from django.utils.translation import gettext_lazy as _
from app_gateway.dns_providers import DNS_PROVIDER_CHOICES
from cluster.models import ClusterSettings
from gatekeeper.models import GatekeeperGroup, AuthMethod, _unique_slug
RESERVED_APP_NAME = 'wireguard_webadmin'
@@ -44,6 +45,9 @@ class Application(models.Model):
display_name = models.CharField(max_length=128, blank=True)
upstream = models.CharField(max_length=255, help_text=_("Upstream address, e.g.: http://10.188.18.27:3000"))
allow_invalid_cert = models.BooleanField(default=False, help_text=_("Allow invalid or self-signed TLS certificates from the upstream"))
serve_on_primary = models.BooleanField(default=True)
all_workers = models.BooleanField(default=False)
workers = models.ManyToManyField('cluster.Worker', blank=True, related_name='applications')
created = models.DateTimeField(auto_now_add=True)
updated = models.DateTimeField(auto_now=True)
@@ -59,6 +63,18 @@ class Application(models.Model):
def __str__(self):
return self.display_name or self.name
# Without an active cluster every application is served by the primary server only
@property
def served_by_workers(self):
return ClusterSettings.cluster_enabled() and (self.all_workers or self.workers.exists())
def is_served_by(self, worker):
if not ClusterSettings.cluster_enabled():
return worker is None
if worker is None:
return self.serve_on_primary
return self.all_workers or self.workers.filter(pk=worker.pk).exists()
class Meta:
ordering = ['name']
+306 -2
View File
@@ -8,13 +8,20 @@ from pathlib import Path
from unittest.mock import patch
from django.conf import settings
from django.test import TestCase
from django.contrib.auth.models import User
from django.test import TestCase, override_settings
from app_gateway.caddy_config_export import export_caddy_config
from app_gateway.models import AccessPolicy, Application, ApplicationHost, ApplicationPolicy, ApplicationRoute
from app_gateway.forms import ApplicationForm, ApplicationHostForm
from app_gateway.models import (
AccessPolicy, Application, ApplicationHost, ApplicationPolicy, ApplicationRoute, DnsChallengeCredential
)
from app_gateway.setup_defaults import create_default_entries
from cluster.models import ClusterSettings, Worker
from gatekeeper.models import (
AuthMethod, AuthMethodAllowedDomain, AuthMethodAllowedEmail, GatekeeperGroup, GatekeeperIPAddress, GatekeeperUser
)
from user_manager.models import UserAcl
AUTH_GATEWAY_FIXTURES_DIR = Path(settings.BASE_DIR) / 'containers' / 'auth-gateway' / 'tests' / 'fixtures' / 'config_files'
UPDATE_FIXTURES_ENV = 'UPDATE_AUTH_GATEWAY_FIXTURES'
@@ -29,6 +36,11 @@ _WEBADMIN_EXPORT_SPEC = importlib.util.spec_from_file_location('caddy_export_wir
_WEBADMIN_EXPORT_MODULE = importlib.util.module_from_spec(_WEBADMIN_EXPORT_SPEC)
_WEBADMIN_EXPORT_SPEC.loader.exec_module(_WEBADMIN_EXPORT_MODULE)
_PROCESS_CONFIG_PATH = Path(settings.BASE_DIR) / 'containers' / 'caddy' / 'process_config.py'
_PROCESS_CONFIG_SPEC = importlib.util.spec_from_file_location('caddy_process_config', _PROCESS_CONFIG_PATH)
process_config = importlib.util.module_from_spec(_PROCESS_CONFIG_SPEC)
_PROCESS_CONFIG_SPEC.loader.exec_module(process_config)
def create_auth_gateway_scenario():
local_password = AuthMethod.objects.create(display_name='Local Password', auth_type='local_password')
@@ -106,10 +118,16 @@ def export_auth_gateway_fixtures(output_dir):
_WEBADMIN_EXPORT_MODULE._write_config(os.path.join(output_dir, 'wireguard_webadmin.json'), webadmin_config)
# Same hosts used for wireguard_webadmin.json, so the WireGuard WebAdmin hosts do not depend on the local .env
FIXTURE_ALLOWED_HOSTS = ['wireguard-webadmin', 'webadmin.example.test', 'webadmin2.example.test', '10.0.0.1']
@override_settings(ALLOWED_HOSTS=FIXTURE_ALLOWED_HOSTS)
class AuthGatewayFixturesTests(TestCase):
maxDiff = None
def setUp(self):
create_default_entries()
create_auth_gateway_scenario()
def test_auth_gateway_fixtures_match_exporter(self):
@@ -131,3 +149,289 @@ class AuthGatewayFixturesTests(TestCase):
f"{filename} is out of date. Regenerate it with: "
f"{UPDATE_FIXTURES_ENV}=1 python manage.py test app_gateway"
)
CLOUDFLARE_TOKEN = 'cfut_' + 'a' * 40
def caddy_app(app_id, hosts):
return {'id': app_id, 'hosts': hosts, 'upstream': 'http://10.0.0.10:8080'}
def site_block(caddyfile, host):
"""Return the lines of the site block that starts with the given host."""
lines = caddyfile.splitlines()
start = next(index for index, line in enumerate(lines) if line.startswith(host) and line.endswith('{'))
end = next(index for index in range(start + 1, len(lines)) if lines[index] == '}')
return lines[start:end + 1]
class CaddyfileTlsTests(TestCase):
tls_data = {
'hosts': {
'http.example.test': {'certificate_type': 'http_challenge'},
'strict.example.test': {'certificate_type': 'http_challenge', 'certificate_fallback': False},
'dns.example.test': {'certificate_type': 'dns_challenge', 'dns_credential': 'cloudflare'},
'internal.example.test': {'certificate_type': 'self_signed'},
'worker-only.example.test': {'certificate_type': 'dns_challenge', 'dns_credential': 'cloudflare',
'certificate_only': True, 'export_certificate': True},
},
'dns_credentials': {'cloudflare': {'provider': 'cloudflare', 'credentials': {'api_token': CLOUDFLARE_TOKEN}}},
}
def build(self, apps, tls_data=None):
return process_config.build_caddyfile(apps, None, None, tls_data or self.tls_data)
def test_hosts_are_grouped_by_certificate_settings(self):
caddyfile = self.build([caddy_app('app', [
'http.example.test', 'strict.example.test', 'dns.example.test', 'internal.example.test', 'unknown.example.test'
])])
self.assertEqual(
site_block(caddyfile, 'http.example.test')[:5],
['http.example.test, unknown.example.test {', ' tls {', ' issuer acme', ' issuer internal', ' }'],
)
self.assertEqual(site_block(caddyfile, 'strict.example.test')[:4], ['strict.example.test {', ' tls {', ' issuer acme', ' }'])
self.assertEqual(site_block(caddyfile, 'internal.example.test')[1], ' tls internal')
dns_block = '\n'.join(site_block(caddyfile, 'dns.example.test'))
self.assertIn(f'api_token "{CLOUDFLARE_TOKEN}"', dns_block)
self.assertIn('propagation_delay 30s', dns_block)
self.assertIn('propagation_timeout -1', dns_block)
self.assertIn('issuer internal', dns_block)
def test_dns_challenge_can_be_disabled_after_invalid_configuration(self):
with patch.object(process_config, 'DISABLE_DNS_CHALLENGE', True):
caddyfile = self.build([caddy_app('app', ['dns.example.test'])])
self.assertNotIn('dns cloudflare', caddyfile)
self.assertIn('issuer acme', '\n'.join(site_block(caddyfile, 'dns.example.test')))
def test_primary_keeps_certificates_of_worker_only_hosts(self):
caddyfile = self.build([caddy_app('app', ['http.example.test'])])
worker_only_block = site_block(caddyfile, 'worker-only.example.test')
self.assertIn(' respond 404', worker_only_block)
self.assertIn(' dns cloudflare {', worker_only_block)
def test_cluster_node_never_requests_acme_certificates(self):
node_tls = {'hosts': {
'ready.example.test': {'certificate_type': 'dns_challenge', 'certificate_file': 'a' * 64},
'waiting.example.test': {'certificate_type': 'dns_challenge', 'certificate_file': ''},
'strict.example.test': {'certificate_type': 'dns_challenge', 'certificate_fallback': False, 'certificate_file': ''},
'internal.example.test': {'certificate_type': 'self_signed'},
}, 'dns_credentials': {}}
apps = [caddy_app('app', list(node_tls['hosts']))]
with patch.object(process_config, 'CLUSTER_NODE', True), \
patch.object(process_config, 'NODE_CERTIFICATES_DIR', '/caddy_certificates'):
caddyfile = self.build(apps, node_tls)
self.assertNotIn('issuer acme', caddyfile)
self.assertNotIn('strict.example.test', caddyfile)
certificate_path = '/caddy_certificates/' + 'a' * 64
self.assertEqual(site_block(caddyfile, 'ready.example.test')[1], f' tls {certificate_path}.crt {certificate_path}.key')
self.assertEqual(site_block(caddyfile, 'waiting.example.test')[0], 'waiting.example.test, internal.example.test {')
self.assertEqual(site_block(caddyfile, 'waiting.example.test')[1], ' tls internal')
class CaddyTlsStateTests(TestCase):
"""Fallback certificates are removed and Caddy is restarted only when the certificate settings change."""
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.addCleanup(self.temp_dir.cleanup)
self.certificates_dir = os.path.join(self.temp_dir.name, 'certificates')
self.json_dir = os.path.join(self.temp_dir.name, 'json')
self.export_dir = os.path.join(self.temp_dir.name, 'export')
for directory in (self.certificates_dir, self.json_dir, self.export_dir):
os.makedirs(directory)
for attribute, value in (
('CERTIFICATES_DIR', self.certificates_dir),
('TLS_STATE_PATH', os.path.join(self.temp_dir.name, 'tls_state.json')),
('TLS_STATE_PENDING_PATH', os.path.join(self.temp_dir.name, 'tls_state_pending.json')),
('JSON_DIR', self.json_dir),
('CERTIFICATE_EXPORT_DIR', self.export_dir),
):
patcher = patch.object(process_config, attribute, value)
patcher.start()
self.addCleanup(patcher.stop)
def store_certificate(self, issuer_dir_name, host, content='certificate'):
host_dir = os.path.join(self.certificates_dir, issuer_dir_name, host)
os.makedirs(host_dir, exist_ok=True)
for extension in ('crt', 'key'):
with open(os.path.join(host_dir, f'{host}.{extension}'), 'w', encoding='utf-8') as certificate_file:
certificate_file.write(f'{content}-{extension}')
return host_dir
def apply(self, host_tls):
apps = [caddy_app('app', ['app.example.test'])]
tls_data = {'hosts': {'app.example.test': host_tls}, 'dns_credentials': {}}
process_config._write_json(process_config.TLS_STATE_PENDING_PATH, process_config.build_tls_state(apps, tls_data))
output = io.StringIO()
with contextlib.redirect_stdout(output), contextlib.redirect_stderr(io.StringIO()):
process_config.sync_tls_state()
return output.getvalue().strip() == 'restart'
def test_first_run_never_restarts(self):
fallback_dir = self.store_certificate('local', 'app.example.test')
self.assertFalse(self.apply({'certificate_type': 'http_challenge'}))
self.assertTrue(os.path.isdir(fallback_dir))
def test_retry_removes_fallback_certificate(self):
self.apply({'certificate_type': 'http_challenge'})
fallback_dir = self.store_certificate('local', 'app.example.test')
self.assertTrue(self.apply({'certificate_type': 'http_challenge', 'certificate_generation': 1}))
self.assertFalse(os.path.isdir(fallback_dir))
def test_retry_keeps_valid_acme_certificate_without_restart(self):
self.apply({'certificate_type': 'http_challenge'})
acme_dir = self.store_certificate('acme-v02.api.letsencrypt.org-directory', 'app.example.test')
self.assertFalse(self.apply({'certificate_type': 'http_challenge', 'certificate_generation': 1}))
self.assertTrue(os.path.isdir(acme_dir))
def test_switching_to_self_signed_restarts_when_acme_certificate_is_cached(self):
self.apply({'certificate_type': 'http_challenge'})
self.store_certificate('acme-v02.api.letsencrypt.org-directory', 'app.example.test')
self.assertTrue(self.apply({'certificate_type': 'self_signed'}))
def test_unchanged_settings_do_not_restart(self):
self.apply({'certificate_type': 'http_challenge'})
self.store_certificate('local', 'app.example.test')
self.assertFalse(self.apply({'certificate_type': 'http_challenge'}))
def test_readded_host_with_leftover_fallback_is_retried(self):
self.apply({'certificate_type': 'http_challenge'})
fallback_dir = self.store_certificate('local', 'other.example.test')
apps = [caddy_app('app', ['app.example.test', 'other.example.test'])]
tls_data = {'hosts': {'app.example.test': {'certificate_type': 'http_challenge'},
'other.example.test': {'certificate_type': 'dns_challenge', 'dns_credential': 'cloudflare'}},
'dns_credentials': {'cloudflare': {'provider': 'cloudflare', 'credentials': {'api_token': CLOUDFLARE_TOKEN}}}}
process_config._write_json(process_config.TLS_STATE_PENDING_PATH, process_config.build_tls_state(apps, tls_data))
output = io.StringIO()
with contextlib.redirect_stdout(output), contextlib.redirect_stderr(io.StringIO()):
process_config.sync_tls_state()
self.assertEqual(output.getvalue().strip(), 'restart')
self.assertFalse(os.path.isdir(fallback_dir))
def test_new_host_without_cached_certificate_does_not_restart(self):
self.apply({'certificate_type': 'http_challenge'})
apps = [caddy_app('app', ['app.example.test', 'new.example.test'])]
tls_data = {'hosts': {'app.example.test': {'certificate_type': 'http_challenge'},
'new.example.test': {'certificate_type': 'http_challenge'}}, 'dns_credentials': {}}
process_config._write_json(process_config.TLS_STATE_PENDING_PATH, process_config.build_tls_state(apps, tls_data))
output = io.StringIO()
with contextlib.redirect_stdout(output), contextlib.redirect_stderr(io.StringIO()):
process_config.sync_tls_state()
self.assertEqual(output.getvalue().strip(), '')
def test_cluster_node_restarts_when_certificate_changes(self):
with patch.object(process_config, 'CLUSTER_NODE', True):
self.apply({'certificate_type': 'dns_challenge', 'certificate_file': ''})
fallback_dir = self.store_certificate('local', 'app.example.test')
self.assertTrue(self.apply({'certificate_type': 'dns_challenge', 'certificate_file': 'a' * 64}))
self.assertFalse(os.path.isdir(fallback_dir))
self.assertTrue(self.apply({'certificate_type': 'dns_challenge', 'certificate_file': 'b' * 64}))
def test_only_acme_certificates_are_exported(self):
tls_data = {'hosts': {
'exported.example.test': {'certificate_type': 'dns_challenge', 'export_certificate': True},
'fallback.example.test': {'certificate_type': 'dns_challenge', 'export_certificate': True},
'local.example.test': {'certificate_type': 'dns_challenge'},
}}
with open(os.path.join(self.json_dir, 'tls.json'), 'w', encoding='utf-8') as tls_file:
json.dump(tls_data, tls_file)
self.store_certificate('acme-v02.api.letsencrypt.org-directory', 'exported.example.test', 'acme')
self.store_certificate('acme-v02.api.letsencrypt.org-directory', 'local.example.test', 'acme')
self.store_certificate('local', 'fallback.example.test', 'internal')
os.makedirs(os.path.join(self.export_dir, 'removed.example.test'))
with contextlib.redirect_stdout(io.StringIO()):
process_config.export_certificates()
self.assertEqual(sorted(os.listdir(self.export_dir)), ['exported.example.test'])
with open(os.path.join(self.export_dir, 'exported.example.test', 'certificate.crt'), encoding='utf-8') as certificate_file:
self.assertEqual(certificate_file.read(), 'acme-crt')
@override_settings(ALLOWED_HOSTS=FIXTURE_ALLOWED_HOSTS)
class ClusterApplicationFormTests(TestCase):
def setUp(self):
ClusterSettings.objects.update_or_create(name='cluster_settings', defaults={'enabled': True})
self.worker = Worker.objects.create(name='node-a')
self.application = Application.objects.create(display_name='Wiki', upstream='http://10.0.0.30:8080')
self.credential = DnsChallengeCredential.objects.create(
display_name='Cloudflare', provider='cloudflare', credentials={'api_token': CLOUDFLARE_TOKEN}
)
def test_application_with_http_validation_hosts_cannot_be_served_by_workers(self):
ApplicationHost.objects.create(application=self.application, hostname='wiki.example.test')
form = ApplicationForm(data={
'display_name': 'Wiki', 'upstream': 'http://10.0.0.30:8080', 'serve_on_primary': 'on',
'workers': [self.worker.pk],
}, instance=self.application)
self.assertFalse(form.is_valid())
self.assertIn('wiki.example.test', str(form.errors))
def test_all_workers_and_specific_workers_are_exclusive(self):
form = ApplicationForm(data={
'display_name': 'Wiki', 'upstream': 'http://10.0.0.30:8080', 'serve_on_primary': 'on',
'all_workers': 'on', 'workers': [self.worker.pk],
}, instance=self.application)
self.assertIn('workers', form.errors)
def test_application_must_be_published_somewhere(self):
form = ApplicationForm(data={'display_name': 'Wiki', 'upstream': 'http://10.0.0.30:8080'}, instance=self.application)
self.assertFalse(form.is_valid())
self.assertIn('__all__', form.errors)
def test_without_cluster_everything_is_served_by_the_primary(self):
self.application.serve_on_primary = False
self.application.save()
self.application.workers.add(self.worker)
ClusterSettings.objects.filter(name='cluster_settings').update(enabled=False)
self.assertTrue(self.application.is_served_by(None))
self.assertFalse(self.application.is_served_by(self.worker))
self.assertFalse(self.application.served_by_workers)
form = ApplicationForm(data={'display_name': 'Wiki', 'upstream': 'http://10.0.0.30:8080'}, instance=self.application)
self.assertTrue(form.is_valid(), form.errors)
self.assertNotIn('workers', form.fields)
def test_host_of_application_served_by_workers_requires_dns_or_self_signed(self):
self.application.workers.add(self.worker)
form = ApplicationHostForm(
data={'hostname': 'wiki.example.test', 'certificate_type': 'http_challenge', 'certificate_fallback': 'on'},
application=self.application,
)
self.assertIn('certificate_type', form.errors)
form = ApplicationHostForm(data={
'hostname': 'wiki.example.test', 'certificate_type': 'dns_challenge',
'dns_credential': self.credential.pk, 'certificate_fallback': 'on',
}, application=self.application)
self.assertTrue(form.is_valid(), form.errors)
@override_settings(ALLOWED_HOSTS=FIXTURE_ALLOWED_HOSTS + ['testserver'], CADDY_ENABLED=True, DEBUG=False)
class GatewayAutoApplyTests(TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.addCleanup(self.temp_dir.cleanup)
patcher = patch('app_gateway.caddy_config_export.CADDY_EXPORT_DIR', self.temp_dir.name)
patcher.start()
self.addCleanup(patcher.stop)
user = User.objects.create_user(username='gateway-admin', password='unused-test-password')
UserAcl.objects.create(user=user, user_level=50)
self.client.force_login(user)
def save_application(self):
return self.client.post('/app_gateway/application/manage/', {
'display_name': 'Auto Apply', 'upstream': 'http://10.0.0.40:8080',
})
@override_settings(AUTO_APPLY=True)
def test_saving_applies_configuration(self):
self.save_application()
with open(os.path.join(self.temp_dir.name, 'applications.json'), encoding='utf-8') as applications_file:
self.assertIn('auto-apply', [entry['id'] for entry in json.load(applications_file)['entries']])
@override_settings(AUTO_APPLY=False)
def test_disabled_auto_apply_keeps_changes_pending(self):
self.save_application()
self.assertFalse(os.path.exists(os.path.join(self.temp_dir.name, 'applications.json')))
+104 -33
View File
@@ -1,4 +1,4 @@
import os
from datetime import timedelta
from django.conf import settings
from django.contrib import messages
@@ -7,13 +7,18 @@ from django.db.models import F, ProtectedError
from django.http import JsonResponse
from django.shortcuts import render, get_object_or_404, redirect
from django.urls import reverse
from django.utils import timezone
from django.utils.dateparse import parse_datetime
from django.utils.formats import date_format
from django.utils.http import url_has_allowed_host_and_scheme
from django.utils.timezone import localtime
from django.utils.translation import gettext as _
from django.views.decorators.http import require_POST
from app_gateway.caddy_config_export import CADDY_EXPORT_DIR, caddy_export_pending, export_caddy_config
from app_gateway.caddy_config_export import (
ACME_CERTIFICATE_TYPES, CADDY_EXPORT_DIR, apply_caddy_config, auto_apply_caddy_config, caddy_export_pending,
load_exported_files, load_worker_export
)
from app_gateway.certificate_status import inspect_certificate
from app_gateway.forms import (
ApplicationForm, ApplicationHostForm, AccessPolicyForm,
@@ -24,6 +29,7 @@ from app_gateway.models import (
RESERVED_APP_NAME
)
from app_gateway.setup_defaults import create_default_entries
from cluster.models import ClusterSettings, Worker
from user_manager.models import UserAcl
@@ -79,6 +85,36 @@ def view_application_details(request):
return render(request, 'app_gateway/application_details.html', context)
def worker_runs_gateway(worker):
# Workers running the App Gateway report the Caddy service; VPN-only workers do not serve applications
worker_status = getattr(worker, 'workerstatus', None)
return bool(worker_status and 'caddy' in (worker_status.services_status or {}))
def describe_certificate_status(certificate, certificate_type):
"""Badge information for a certificate inspected on the primary server or reported by a cluster worker."""
expiration = ''
if certificate.get('not_after'):
not_after = certificate['not_after']
if isinstance(not_after, str):
not_after = parse_datetime(not_after)
if not_after:
expiration = _('expires %(date)s') % {'date': date_format(localtime(not_after), 'SHORT_DATETIME_FORMAT')}
if certificate.get('status') == 'trusted':
return 'success', _('Trusted'), f"{certificate.get('issuer', '')}, {expiration}"
if certificate.get('status') == 'untrusted' and certificate_type == 'self_signed':
return 'secondary', _('Self-signed'), expiration
if certificate.get('status') == 'untrusted':
detail = _('The trusted certificate could not be issued yet, %(expiration)s') % {'expiration': expiration}
return 'warning', _('Using fallback'), detail
if certificate.get('status') == 'unknown':
return 'light', _('Unknown'), _('No recent report from this server.')
if certificate.get('status') == 'pending_export':
return 'info', _('Pending export'), _('Export the configuration to publish this host on this server.')
return 'danger', _('Unavailable'), _('No certificate is being served for this hostname.')
@login_required
def view_application_certificate_status(request):
if not UserAcl.objects.filter(user=request.user).filter(user_level__gte=20).exists():
@@ -88,29 +124,53 @@ def view_application_certificate_status(request):
if not settings.CADDY_ENABLED:
return JsonResponse({'caddy_enabled': False, 'hosts': {}})
workers = []
if application.name != RESERVED_APP_NAME and ClusterSettings.objects.filter(name='cluster_settings', enabled=True).exists():
workers = [
worker for worker in Worker.objects.filter(enabled=True).select_related('workerstatus').order_by('name')
if application.is_served_by(worker) and worker_runs_gateway(worker)
]
report_limit = timezone.now() - timedelta(minutes=10)
# Servers only publish what was exported, so hosts missing from their exported files are pending
primary_export = load_exported_files(CADDY_EXPORT_DIR) or {}
primary_exported_hosts = primary_export.get('tls.json', {}).get('hosts', {})
worker_exported_hosts = {}
for worker in workers:
worker_export = load_worker_export(worker) or {}
worker_exported_hosts[worker.pk] = worker_export.get('tls.json', {}).get('hosts', {})
hosts_status = {}
for application_host in application.hosts.all():
certificate = inspect_certificate(application_host.hostname)
expiration = ''
if certificate['not_after']:
expiration = _('expires %(date)s') % {
'date': date_format(localtime(certificate['not_after']), 'SHORT_DATETIME_FORMAT')
}
hostname = application_host.hostname.lower()
servers = []
# The primary server also manages ACME certificates of applications only served by workers
primary_handles_host = application.is_served_by(None) or (
application_host.certificate_type in ACME_CERTIFICATE_TYPES and application.served_by_workers
)
if primary_handles_host:
if hostname in primary_exported_hosts:
certificate = inspect_certificate(application_host.hostname)
else:
certificate = {'status': 'pending_export'}
servers.append((_('Primary Server'), certificate))
if certificate['status'] == 'trusted':
css_class, label = 'success', _('Trusted')
detail = f"{certificate['issuer']}, {expiration}"
elif certificate['status'] == 'untrusted' and application_host.certificate_type == 'self_signed':
css_class, label = 'secondary', _('Self-signed')
detail = expiration
elif certificate['status'] == 'untrusted':
css_class, label = 'warning', _('Using fallback')
detail = _('The trusted certificate could not be issued yet, %(expiration)s') % {'expiration': expiration}
else:
css_class, label = 'danger', _('Unavailable')
detail = _('No certificate is being served for this hostname.')
for worker in workers:
worker_status = getattr(worker, 'workerstatus', None)
if hostname not in worker_exported_hosts[worker.pk]:
certificate = {'status': 'pending_export'}
elif worker_status and worker_status.certificate_status_updated and worker_status.certificate_status_updated > report_limit:
certificate = worker_status.certificate_status.get(hostname, {'status': 'unavailable'})
else:
certificate = {'status': 'unknown'}
servers.append((worker.display_name, certificate))
hosts_status[str(application_host.uuid)] = {'css_class': css_class, 'label': label, 'detail': detail}
hosts_status[str(application_host.uuid)] = []
for server_name, certificate in servers:
css_class, label, detail = describe_certificate_status(certificate, application_host.certificate_type)
hosts_status[str(application_host.uuid)].append({
'server': server_name if workers else '', 'css_class': css_class, 'label': label, 'detail': detail,
})
return JsonResponse({'caddy_enabled': True, 'hosts': hosts_status})
@@ -132,13 +192,17 @@ def view_manage_application(request):
application = None
title = _('Create Application')
cancel_url = reverse('app_gateway_list') + '?tab=applications'
if application:
cancel_url = reverse('view_application') + f'?uuid={application.uuid}'
else:
cancel_url = reverse('app_gateway_list') + '?tab=applications'
form = ApplicationForm(request.POST or None, instance=application, cancel_url=cancel_url)
if form.is_valid():
form.save()
application = form.save()
messages.success(request, _('Application saved successfully.'))
return redirect(cancel_url)
auto_apply_caddy_config(request)
return redirect(reverse('view_application') + f'?uuid={application.uuid}')
form_description = {
'size': 'col-lg-6',
@@ -178,6 +242,7 @@ def view_delete_application(request):
if request.method == 'POST':
application.delete()
messages.success(request, _('Application deleted successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
context = {
@@ -214,13 +279,15 @@ def view_manage_application_host(request):
return redirect(cancel_url)
form = ApplicationHostForm(
request.POST or None, instance=application_host, cancel_url=cancel_url, is_reserved=is_reserved
request.POST or None, instance=application_host, cancel_url=cancel_url, is_reserved=is_reserved,
application=application,
)
if form.is_valid():
host = form.save(commit=False)
host.application = application
host.save()
messages.success(request, _('Application Host saved successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
form_description_content = _('''
@@ -233,6 +300,7 @@ def view_manage_application_host(request):
</ul>
<p>When the fallback option is enabled and a trusted certificate cannot be issued, a self-signed certificate is used until the next successful attempt. Without the fallback, the connection fails until the certificate is issued.</p>
<p>Use <strong>Retry certificate</strong> in the host list to request a new attempt right away, for example after fixing the DNS or firewall.</p>
<p><strong>Important:</strong> the certificate is requested as soon as the configuration is applied. With HTTP validation, the hostname must already point to this server: failed attempts count towards the Let's Encrypt limit (5 per hour per hostname). If the DNS is not ready yet, use DNS validation.</p>
''')
if is_reserved:
form_description_content += _('''
@@ -268,6 +336,7 @@ def view_delete_application_host(request):
if request.method == 'POST':
application_host.delete()
messages.success(request, _('Application Host deleted successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
context = {
@@ -312,6 +381,7 @@ def view_manage_access_policy(request):
if form.is_valid():
form.save()
messages.success(request, _('Access Policy saved successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
if policy_type == 'public':
@@ -365,6 +435,7 @@ def view_delete_access_policy(request):
try:
access_policy.delete()
messages.success(request, _('Access Policy deleted successfully.'))
auto_apply_caddy_config(request)
except ProtectedError:
messages.error(request, _('Cannot delete this Access Policy because it is currently in use by an Application Route or Application Default Policy.'))
return redirect(cancel_url)
@@ -403,6 +474,7 @@ def view_manage_application_policy(request):
policy_config.application = application
policy_config.save()
messages.success(request, _('Application Default Policy saved successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
context = {
@@ -426,6 +498,7 @@ def view_delete_application_policy(request):
if request.method == 'POST':
application_policy.delete()
messages.success(request, _('Application Default Policy deleted successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
context = {
@@ -464,6 +537,7 @@ def view_manage_application_route(request):
route.application = application
route.save()
messages.success(request, _('Application Route saved successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
form_description = {
@@ -502,6 +576,7 @@ def view_delete_application_route(request):
if request.method == 'POST':
application_route.delete()
messages.success(request, _('Application Route deleted successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
context = {
@@ -534,9 +609,7 @@ def view_retry_application_host_certificate(request):
return redirect(redirect_url)
ApplicationHost.objects.filter(pk=application_host.pk).update(certificate_generation=F('certificate_generation') + 1)
export_caddy_config(CADDY_EXPORT_DIR)
if settings.DEBUG:
export_caddy_config(os.path.join(settings.BASE_DIR, 'containers', 'caddy', 'config_files'))
apply_caddy_config()
messages.success(request, _(
'A new certificate will be requested for %(hostname)s. The current configuration was exported.'
@@ -564,6 +637,7 @@ def view_manage_dns_credential(request):
if form.is_valid():
form.save()
messages.success(request, _('DNS Credential saved successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
context = {
@@ -587,6 +661,7 @@ def view_delete_dns_credential(request):
try:
dns_credential.delete()
messages.success(request, _('DNS Credential deleted successfully.'))
auto_apply_caddy_config(request)
except ProtectedError:
messages.error(request, _('Cannot delete this DNS Credential because it is currently in use by an Application Host.'))
return redirect(cancel_url)
@@ -618,10 +693,6 @@ def view_export_caddy_config(request):
))
return redirect(redirect_url)
export_caddy_config(CADDY_EXPORT_DIR)
if settings.DEBUG:
export_caddy_config(os.path.join(settings.BASE_DIR, 'containers', 'caddy', 'config_files'))
apply_caddy_config()
messages.success(request, _('Configuration exported successfully.'))
return redirect(redirect_url)
+5 -5
View File
@@ -5,18 +5,18 @@ from .models import ClusterSettings, Worker, WorkerStatus
@admin.register(ClusterSettings)
class ClusterSettingsAdmin(admin.ModelAdmin):
list_display = ('name', 'enabled', 'cluster_mode', 'restart_mode', 'config_version', 'created', 'updated')
list_filter = ('enabled', 'cluster_mode', 'restart_mode', 'primary_enable_wireguard')
list_display = ('name', 'enabled', 'cluster_mode', 'created', 'updated')
list_filter = ('enabled', 'cluster_mode', 'primary_enable_wireguard')
@admin.register(Worker)
class WorkerAdmin(admin.ModelAdmin):
list_display = ('name', 'enabled', 'ip_address', 'country', 'city', 'ip_lock', 'created', 'updated')
list_filter = ('enabled', 'ip_lock', 'country', 'force_reload', 'force_restart')
list_filter = ('enabled', 'ip_lock', 'country')
@admin.register(WorkerStatus)
class WorkerStatusAdmin(admin.ModelAdmin):
list_display = ('worker', 'last_seen', 'config_version', 'last_reload', 'last_restart')
list_filter = ('last_seen', 'last_reload', 'last_restart')
list_display = ('worker', 'last_seen', 'worker_version', 'wireguard_status_updated')
list_filter = ('last_seen',)
+132 -182
View File
@@ -1,220 +1,170 @@
import glob
import json
import os
from django.conf import settings
from django.http import JsonResponse, FileResponse
from django.core.exceptions import ValidationError
from django.http import JsonResponse
from django.utils import timezone
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.gzip import gzip_page
from django.views.decorators.http import require_GET, require_POST
from .models import ClusterSettings, Worker, WorkerStatus
from cluster.models import ClusterSettings, Worker, WorkerStatus
from cluster.sync import SECTION_NAMES, build_manifest, build_section
REPORT_STATUS_FIELDS = ('status', 'message')
def get_ip_address(request):
ip_address = request.META.get('HTTP_X_FORWARDED_FOR')
if ip_address:
ip_address = ip_address.split(',')[0]
ip_address = ip_address.split(',')[0].strip()
else:
ip_address = request.META.get('REMOTE_ADDR')
return ip_address
def get_cluster_settings():
cluster_settings, created = ClusterSettings.objects.get_or_create(name='cluster_settings')
return {
'enabled': cluster_settings.enabled,
'primary_enable_wireguard': cluster_settings.primary_enable_wireguard,
'stats_sync_interval': settings.WIREGUARD_STATUS_CACHE_REFRESH_INTERVAL,
'cluster_mode': cluster_settings.cluster_mode,
'restart_mode': cluster_settings.restart_mode,
'config_version': cluster_settings.config_version,
'dns_version': cluster_settings.dns_version,
}
def _error_response(message, status):
return JsonResponse({'status': 'error', 'message': message}, status=status)
def get_worker(request):
success = True
ip_address = get_ip_address(request)
token = request.GET.get('token', '')
"""
Authenticate the worker from the Authorization header and update its status.
Returns (worker, None) on success or (None, JsonResponse) when the request must be rejected.
"""
authorization = request.headers.get('Authorization', '')
if not authorization.startswith('Bearer '):
return None, _error_response('Worker not found', 403)
try:
worker = Worker.objects.get(token=token)
except:
return None, False
worker = Worker.objects.get(token=authorization[len('Bearer '):].strip())
except (Worker.DoesNotExist, ValidationError, ValueError):
return None, _error_response('Worker not found', 403)
worker_status, created = WorkerStatus.objects.get_or_create(worker=worker)
worker_status, worker_status_created = WorkerStatus.objects.get_or_create(worker=worker)
try:
worker_config_version = int(request.GET.get('worker_config_version'))
worker_version = int(request.GET.get('worker_version'))
worker_dns_version = int(request.GET.get('worker_dns_version'))
except:
worker.error_status = 'missing_version'
worker.save()
return worker, False
if worker.error_status == 'missing_version':
worker.error_status = ''
worker.save()
if worker_version < settings.CLUSTER_WORKER_MINIMUM_VERSION:
worker.error_status = 'update_required'
worker.save()
return worker, False
if worker.error_status == 'update_required':
worker.error_status = ''
worker.save()
if worker_status.config_version != worker_config_version:
worker_status.config_version = worker_config_version
if worker_status.worker_version != worker_version:
worker_status.worker_version = worker_version
if worker_status.dns_version != worker_dns_version:
worker_status.dns_version = worker_dns_version
worker_version = int(request.headers.get('X-Worker-Version', '0'))
except ValueError:
worker_version = 0
worker_status.worker_version = worker_version
worker_status.last_seen = timezone.now()
worker_status.save()
if not worker.ip_address:
worker_status.save(update_fields=['worker_version', 'last_seen', 'updated'])
ip_address = get_ip_address(request)
error_status = ''
if worker_version < settings.CLUSTER_WORKER_MINIMUM_VERSION:
error_status = 'update_required'
elif worker.ip_lock and worker.ip_address and worker.ip_address != ip_address:
error_status = 'ip_lock'
elif not worker.enabled:
error_status = 'worker_disabled'
else:
cluster_settings, cluster_settings_created = ClusterSettings.objects.get_or_create(name='cluster_settings')
if cluster_settings.enabled and not settings.WIREGUARD_STATUS_CACHE_ENABLED:
cluster_settings.enabled = False
cluster_settings.save()
if not cluster_settings.enabled:
error_status = 'cluster_disabled'
update_fields = []
if not worker.ip_lock and worker.ip_address != ip_address:
worker.ip_address = ip_address
worker.save()
update_fields.append('ip_address')
if worker.error_status != error_status:
worker.error_status = error_status
update_fields.append('error_status')
if update_fields:
worker.save(update_fields=update_fields + ['updated'])
if worker.ip_lock:
if worker.ip_address == ip_address:
if worker.error_status == 'ip_lock':
worker.error_status = ''
worker.save()
else:
worker.error_status = 'ip_lock'
worker.save()
success = False
else:
if worker.ip_address != ip_address:
worker.ip_address = ip_address
worker.save()
if worker.enabled:
if worker.error_status == 'worker_disabled':
worker.error_status = ''
worker.save()
else:
worker.error_status = 'worker_disabled'
worker.save()
success = False
cluster_settings, created = ClusterSettings.objects.get_or_create(name='cluster_settings')
if not settings.WIREGUARD_STATUS_CACHE_ENABLED and cluster_settings.enabled:
cluster_settings.enabled = False
cluster_settings.save()
if cluster_settings.enabled:
if worker.error_status == 'cluster_disabled':
worker.error_status = ''
worker.save()
else:
worker.error_status = 'cluster_disabled'
worker.save()
success = False
return worker, success
if error_status:
return None, _error_response(error_status, 400)
return worker, None
def api_get_worker_dnsmasq_config(request):
dnsmasq_file = "/etc/dnsmasq/dnsmasq_config.tar.gz"
worker, success = get_worker(request)
if worker:
if worker.error_status or not success:
data = {'status': 'error', 'message': worker.error_status}
return JsonResponse(data, status=400)
else:
data = {'status': 'error', 'message': 'Worker not found'}
return JsonResponse(data, status=403)
def api_legacy_cluster_status(request):
# Workers before version 12 poll this endpoint. A 403 makes them shut down WireGuard and stop polling.
return JsonResponse({
'status': 'error',
'message': 'update_required',
'minimum_worker_version': settings.CLUSTER_WORKER_MINIMUM_VERSION,
}, status=403)
if not os.path.exists(dnsmasq_file):
data = {'status': 'error', 'message': 'dnsmasq configuration not found'}
return JsonResponse(data, status=404)
response = FileResponse(open(dnsmasq_file, "rb"), content_type="application/gzip")
response["Content-Disposition"] = 'attachment; filename="dnsmasq_config.tar.gz"'
response["Content-Length"] = str(os.path.getsize(dnsmasq_file))
return response
@require_GET
def api_manifest(request):
worker, error_response = get_worker(request)
if error_response:
return error_response
return JsonResponse({
'status': 'success',
'worker_minimum_version': settings.CLUSTER_WORKER_MINIMUM_VERSION,
'sections': build_manifest(worker),
})
@gzip_page
@require_GET
def api_section(request, section_name):
worker, error_response = get_worker(request)
if error_response:
return error_response
if section_name not in SECTION_NAMES:
return _error_response('Section not found', 404)
return JsonResponse({'status': 'success', 'section': section_name, **build_section(section_name, worker).as_dict()})
def _clean_section_report(sections):
cleaned_sections = {}
for section_name, report in sections.items():
if section_name not in SECTION_NAMES or not isinstance(report, dict):
continue
status = report.get('status')
if status not in ('ok', 'error'):
continue
cleaned_sections[section_name] = {
'hash': str(report.get('hash', ''))[:64],
'status': status,
'message': str(report.get('message', ''))[:1000],
'updated': timezone.now().isoformat(),
}
return cleaned_sections
@csrf_exempt
def api_submit_worker_wireguard_stats(request):
worker, success = get_worker(request)
if worker:
if worker.error_status or not success:
data = {'status': 'error', 'message': worker.error_status}
return JsonResponse(data, status=400)
else:
data = {'status': 'error', 'message': 'Worker not found'}
return JsonResponse(data, status=403)
@require_POST
def api_report(request):
worker, error_response = get_worker(request)
if error_response:
return error_response
try:
if request.method == 'POST':
payload = json.loads(request.body)
worker_status = worker.workerstatus
worker_status.wireguard_status = payload
worker_status.wireguard_status_updated = timezone.now()
worker_status.save()
data = {'status': 'success', 'message': 'Stats received'}
return JsonResponse(data, status=200)
except Exception as e:
pass
payload = json.loads(request.body)
except (ValueError, UnicodeDecodeError):
return _error_response('Invalid JSON', 400)
if not isinstance(payload, dict):
return _error_response('Invalid JSON', 400)
data = {'status': 'error', 'message': 'Stats not received'}
return JsonResponse(data, status=400)
worker_status = worker.workerstatus
update_fields = []
if isinstance(payload.get('sections'), dict):
section_status = dict(worker_status.section_status or {})
section_status.update(_clean_section_report(payload['sections']))
worker_status.section_status = section_status
update_fields.append('section_status')
def api_get_worker_config_files(request):
worker, success = get_worker(request)
if worker:
if worker.error_status or not success:
data = {'status': 'error', 'message': worker.error_status}
return JsonResponse(data, status=400)
else:
data = {'status': 'error', 'message': 'Worker not found'}
return JsonResponse(data, status=403)
if isinstance(payload.get('services'), dict):
worker_status.services_status = payload['services']
update_fields.append('services_status')
config_files = (
glob.glob('/etc/wireguard/wg*.conf') +
glob.glob('/etc/wireguard/wg-firewall.sh')
)
if isinstance(payload.get('certificates'), dict):
worker_status.certificate_status = payload['certificates']
worker_status.certificate_status_updated = timezone.now()
update_fields += ['certificate_status', 'certificate_status_updated']
files = {}
if isinstance(payload.get('wireguard_status'), dict):
worker_status.wireguard_status = payload['wireguard_status']
worker_status.wireguard_status_updated = timezone.now()
update_fields += ['wireguard_status', 'wireguard_status_updated']
for path in config_files:
filename = os.path.basename(path)
with open(path, 'r') as f:
files[filename] = f.read()
data = {'status': 'success', 'files': files, 'cluster_settings': get_cluster_settings()}
return JsonResponse(data, status=200)
def api_worker_ping(request):
worker, success = get_worker(request)
if worker:
if worker.error_status or not success:
data = {'status': 'error', 'message': worker.error_status}
return JsonResponse(data, status=400)
else:
data = {'status': 'error', 'message': 'Worker not found'}
return JsonResponse(data, status=403)
data = {
'status': 'success',
'worker_error_status': worker.error_status,
}
return JsonResponse(data, status=200)
def api_cluster_status(request):
worker, success = get_worker(request)
if worker:
if worker.error_status or not success:
data = {'status': 'error', 'message': worker.error_status}
return JsonResponse(data, status=400)
else:
data = {'status': 'error', 'message': 'Worker not found'}
return JsonResponse(data, status=403)
data = {'status': 'success', 'worker_error_status': worker.error_status, 'cluster_settings': get_cluster_settings()}
return JsonResponse(data, status=200)
if update_fields:
worker_status.save(update_fields=update_fields + ['updated'])
return JsonResponse({'status': 'success'})
+5 -8
View File
@@ -31,8 +31,10 @@ class WorkerForm(forms.ModelForm):
if self.instance.pk:
delete_html = f"<a href='javascript:void(0)' class='btn btn-outline-danger' data-command='delete' onclick='openCommandDialog(this)'>{delete_label}</a>"
back_url = f"/cluster/worker/view/?uuid={self.instance.uuid}"
else:
delete_html = ''
back_url = '/cluster/'
self.helper.layout = Layout(
Row(
@@ -65,7 +67,7 @@ class WorkerForm(forms.ModelForm):
Row(
Column(
Submit('submit', _('Save'), css_class='btn btn-success'),
HTML(f' <a class="btn btn-secondary" href="/cluster/">{back_label}</a> '),
HTML(f' <a class="btn btn-secondary" href="{back_url}">{back_label}</a> '),
HTML(delete_html),
css_class='col-md-12'),
css_class='form-row'
@@ -90,14 +92,13 @@ class WorkerForm(forms.ModelForm):
class ClusterSettingsForm(forms.ModelForm):
class Meta:
model = ClusterSettings
fields = ['enabled', 'primary_enable_wireguard', 'cluster_mode', 'restart_mode', 'worker_display']
fields = ['enabled', 'primary_enable_wireguard', 'cluster_mode', 'worker_display']
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.fields['enabled'].label = _("Cluster Enabled")
self.fields['primary_enable_wireguard'].label = _("Primary Enable WireGuard")
self.fields['cluster_mode'].label = _("Cluster Mode")
self.fields['restart_mode'].label = _("Restart Mode")
self.fields['worker_display'].label = _("Worker Display")
back_label = _("Back")
@@ -112,11 +113,7 @@ class ClusterSettingsForm(forms.ModelForm):
),
Row(
Column('cluster_mode', css_class='form-group col-md-6 mb-0'),
Column('restart_mode', css_class='form-group col-md-6 mb-0'),
css_class='form-row'
),
Row(
Column('worker_display', css_class='form-group col-md-12 mb-0'),
Column('worker_display', css_class='form-group col-md-6 mb-0'),
css_class='form-row'
),
Row(
+78
View File
@@ -0,0 +1,78 @@
# Generated by Django 5.2.19 on 2026-11-19 16:11
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('cluster', '0017_alter_workerstatus_wireguard_status'),
]
operations = [
migrations.RemoveField(
model_name='clustersettings',
name='config_version',
),
migrations.RemoveField(
model_name='clustersettings',
name='dns_version',
),
migrations.RemoveField(
model_name='clustersettings',
name='restart_mode',
),
migrations.RemoveField(
model_name='worker',
name='force_reload',
),
migrations.RemoveField(
model_name='worker',
name='force_restart',
),
migrations.RemoveField(
model_name='workerstatus',
name='active_peers',
),
migrations.RemoveField(
model_name='workerstatus',
name='config_version',
),
migrations.RemoveField(
model_name='workerstatus',
name='dns_version',
),
migrations.RemoveField(
model_name='workerstatus',
name='last_reload',
),
migrations.RemoveField(
model_name='workerstatus',
name='last_restart',
),
migrations.AddField(
model_name='workerstatus',
name='certificate_status',
field=models.JSONField(blank=True, default=dict),
),
migrations.AddField(
model_name='workerstatus',
name='certificate_status_updated',
field=models.DateTimeField(blank=True, null=True),
),
migrations.AddField(
model_name='workerstatus',
name='section_status',
field=models.JSONField(blank=True, default=dict),
),
migrations.AddField(
model_name='workerstatus',
name='services_status',
field=models.JSONField(blank=True, default=dict),
),
migrations.AlterField(
model_name='worker',
name='error_status',
field=models.CharField(choices=[('', ''), ('ip_lock', 'IP lock is enabled, but the worker is attempting to access from a different IP address.'), ('worker_disabled', 'Worker is not enabled'), ('cluster_disabled', 'Cluster is not enabled'), ('update_required', 'Worker update is required.')], default='', max_length=32),
),
]
+10 -12
View File
@@ -9,15 +9,12 @@ class ClusterSettings(models.Model):
enabled = models.BooleanField(default=False)
primary_enable_wireguard = models.BooleanField(default=True)
cluster_mode = models.CharField(default='mirror', max_length=16, choices=(('mirror', 'Mirror'), ))
restart_mode = models.CharField(default='auto', max_length=16, choices=(('auto', 'Automatic restart'),))
worker_display = models.CharField(
default='server_address', max_length=16, choices=(
('name', 'Name'), ('server_address', 'Server Address'),
('location', 'Location'), ('address_location', 'Address + Location')
)
)
config_version = models.PositiveIntegerField(default=0)
dns_version = models.PositiveIntegerField(default=0)
updated = models.DateTimeField(auto_now=True)
created = models.DateTimeField(auto_now_add=True)
@@ -26,6 +23,10 @@ class ClusterSettings(models.Model):
def __str__(self):
return self.name
@classmethod
def cluster_enabled(cls):
return cls.objects.filter(name='cluster_settings', enabled=True).exists()
class Worker(models.Model):
name = models.CharField(max_length=100, unique=True)
@@ -34,9 +35,6 @@ class Worker(models.Model):
ip_lock = models.BooleanField(default=False)
ip_address = models.GenericIPAddressField(blank=True, null=True)
force_reload = models.BooleanField(default=False)
force_restart = models.BooleanField(default=False)
country = models.CharField(max_length=100, blank=True, null=True)
city = models.CharField(max_length=100, blank=True, null=True)
hostname = models.CharField(max_length=100, blank=True, null=True)
@@ -46,7 +44,6 @@ class Worker(models.Model):
('ip_lock', _('IP lock is enabled, but the worker is attempting to access from a different IP address.')),
('worker_disabled', _('Worker is not enabled')),
('cluster_disabled', _('Cluster is not enabled')),
('missing_version', _('Please report worker_config_version, worker_dns_version and worker_version in the API request.')),
('update_required', _('Worker update is required.'))
))
@@ -105,14 +102,15 @@ class Worker(models.Model):
class WorkerStatus(models.Model):
worker = models.OneToOneField(Worker, on_delete=models.CASCADE)
last_seen = models.DateTimeField(auto_now=True)
last_reload = models.DateTimeField(blank=True, null=True)
last_restart = models.DateTimeField(blank=True, null=True)
dns_version = models.PositiveIntegerField(default=0)
config_version = models.PositiveIntegerField(default=0)
worker_version = models.PositiveIntegerField(default=0)
active_peers = models.PositiveIntegerField(default=0)
wireguard_status = models.JSONField(blank=True, null=True)
wireguard_status_updated = models.DateTimeField(blank=True, null=True)
# {section: {"hash": ..., "status": "ok"|"error", "message": ..., "updated": ...}}
section_status = models.JSONField(default=dict, blank=True)
services_status = models.JSONField(default=dict, blank=True)
# {hostname: {"status": ..., "issuer": ..., "not_after": ..., "generation": ...}}
certificate_status = models.JSONField(default=dict, blank=True)
certificate_status_updated = models.DateTimeField(blank=True, null=True)
updated = models.DateTimeField(auto_now=True)
created = models.DateTimeField(auto_now_add=True)
+209
View File
@@ -0,0 +1,209 @@
import glob
import hashlib
import json
import os
from datetime import timedelta
from django.conf import settings
from django.utils import timezone
from django.utils.dateparse import parse_datetime
from app_gateway.caddy_config_export import (
add_certificate_files, certificate_file_name, load_worker_export, read_exported_certificate
)
from cluster.models import WorkerStatus
from dns.functions import generate_dnsmasq_config
from dns.models import DNSFilterList
from firewall.tools import generate_firewall_script
WIREGUARD_CONFIG_DIR = '/etc/wireguard'
DNSMASQ_CONFIG_DIR = '/etc/dnsmasq'
DNSMASQ_MAIN_CONFIG = 'wireguard_webadmin_dns.conf'
# Sections are applied by the node in this order. The firewall comes first because the WireGuard
# configuration runs wg-firewall.sh as PostUp of one of the instances, and certificates come before the
# gateway so the generated Caddyfile only references files that already exist.
SECTION_NAMES = ('firewall', 'wireguard', 'dns', 'settings', 'certificates', 'gateway')
# Large files (e.g. DNS filter lists) are only read and hashed again when they change on disk
_file_cache = {}
def _sha256(text):
return hashlib.sha256(text.encode()).hexdigest()
def _read_file(path):
file_stat = os.stat(path)
cache_key = (file_stat.st_mtime_ns, file_stat.st_size)
cached = _file_cache.get(path)
if cached and cached['key'] == cache_key:
return cached['content'], cached['sha256']
with open(path, 'r', encoding='utf-8', errors='replace') as config_file:
content = config_file.read()
content_sha256 = _sha256(content)
_file_cache[path] = {'key': cache_key, 'content': content, 'sha256': content_sha256}
return content, content_sha256
class Section:
def __init__(self):
self.files = {}
self.file_hashes = {}
self.values = {}
def add_file(self, filename, content, content_sha256=None):
self.files[filename] = content
self.file_hashes[filename] = content_sha256 or _sha256(content)
@property
def hash(self):
fingerprint = json.dumps({'files': self.file_hashes, 'values': self.values}, sort_keys=True)
return _sha256(fingerprint)
def as_dict(self):
return {'hash': self.hash, 'files': self.files, 'values': self.values}
def build_wireguard_section(worker):
section = Section()
for config_path in sorted(glob.glob(os.path.join(WIREGUARD_CONFIG_DIR, 'wg[0-9]*.conf'))):
content, content_sha256 = _read_file(config_path)
section.add_file(os.path.basename(config_path), content, content_sha256)
return section
def build_firewall_section(worker):
section = Section()
section.add_file('wg-firewall.sh', generate_firewall_script(cluster_node=True))
return section
def build_dns_section(worker):
section = Section()
for dns_list in DNSFilterList.objects.filter(enabled=True).order_by('uuid'):
list_path = os.path.join(DNSMASQ_CONFIG_DIR, f'{dns_list.uuid}.conf')
if os.path.isfile(list_path):
content, content_sha256 = _read_file(list_path)
section.add_file(os.path.basename(list_path), content, content_sha256)
section.add_file(DNSMASQ_MAIN_CONFIG, generate_dnsmasq_config(cluster_node=True))
return section
def build_settings_section(worker):
section = Section()
section.values = {
'stats_sync_interval': settings.WIREGUARD_STATUS_CACHE_REFRESH_INTERVAL,
}
return section
EMPTY_GATEWAY_EXPORT = {
'applications.json': {'entries': []},
'auth_policies.json': {'auth_methods': {}, 'groups': {}, 'users': {}, 'policies': {}},
'routes.json': {'entries': {}},
'tls.json': {'hosts': {}, 'dns_credentials': {}},
}
def get_gateway_export(worker):
"""Configuration exported for the worker, with the certificate files currently available on the primary."""
gateway_export = load_worker_export(worker) or json.loads(json.dumps(EMPTY_GATEWAY_EXPORT))
add_certificate_files(gateway_export['tls.json'])
return gateway_export
def build_certificates_section(worker):
"""Certificate and private key of every ACME host served by the worker, named after the certificate hash."""
section = Section()
for hostname, host_tls in sorted(get_gateway_export(worker)['tls.json']['hosts'].items()):
if not host_tls.get('certificate_file'):
continue
certificate, private_key = read_exported_certificate(hostname)
if not certificate or not private_key:
continue
file_name = certificate_file_name(certificate)
section.add_file(f'{file_name}.crt', certificate)
section.add_file(f'{file_name}.key', private_key)
return section
def build_gateway_section(worker):
section = Section()
for file_name, data in get_gateway_export(worker).items():
section.add_file(file_name, json.dumps(data, indent=2) + '\n')
return section
SECTION_BUILDERS = {
'firewall': build_firewall_section,
'wireguard': build_wireguard_section,
'dns': build_dns_section,
'settings': build_settings_section,
'certificates': build_certificates_section,
'gateway': build_gateway_section,
}
def build_section(section_name, worker):
return SECTION_BUILDERS[section_name](worker)
def build_manifest(worker):
return {section_name: build_section(section_name, worker).hash for section_name in SECTION_NAMES}
CERTIFICATE_EXPIRATION_WARNING_DAYS = 14
def get_worker_warnings(worker):
"""Stopped services and certificates that are not trusted or expire soon, as reported by the worker."""
try:
worker_status = worker.workerstatus
except WorkerStatus.DoesNotExist:
return {'stopped_services': [], 'certificate_issues': []}
stopped_services = sorted(
service for service, state in (worker_status.services_status or {}).items() if state != 'running'
)
expiration_limit = timezone.now() + timedelta(days=CERTIFICATE_EXPIRATION_WARNING_DAYS)
certificate_issues = []
for hostname, certificate in sorted((worker_status.certificate_status or {}).items()):
not_after = parse_datetime(certificate.get('not_after') or '')
if certificate.get('status') not in ('trusted', 'untrusted'):
certificate_issues.append(hostname)
elif certificate.get('status') == 'trusted' and not_after and not_after < expiration_limit:
certificate_issues.append(hostname)
return {'stopped_services': stopped_services, 'certificate_issues': certificate_issues}
def get_worker_sync_state(worker, manifest=None):
"""
Compare the hashes reported by the worker with the current ones.
Returns {'state': 'synced'|'pending'|'error', 'sections': {name: {'state': ..., 'message': ...}}}.
"""
manifest = manifest or build_manifest(worker)
try:
section_status = worker.workerstatus.section_status or {}
except WorkerStatus.DoesNotExist:
section_status = {}
sections = {}
for section_name, section_hash in manifest.items():
reported = section_status.get(section_name) or {}
if reported.get('hash') != section_hash:
# A previous error is kept as information while the node retries with the new content
sections[section_name] = {'state': 'pending', 'message': reported.get('message', '')}
elif reported.get('status') == 'error':
sections[section_name] = {'state': 'error', 'message': reported.get('message', '')}
else:
sections[section_name] = {'state': 'synced', 'message': ''}
states = {section['state'] for section in sections.values()}
if 'error' in states:
state = 'error'
elif 'pending' in states:
state = 'pending'
else:
state = 'synced'
return {'state': state, 'sections': sections}
+273 -1
View File
@@ -1 +1,273 @@
# Create your tests here.
import json
import os
import tempfile
from unittest.mock import patch
from django.test import TestCase, override_settings
from django.utils import timezone
from api.models import WireguardStatusCache
from api.views import func_concatenate_cluster_wireguard_status_cache
from app_gateway import caddy_config_export
from app_gateway.caddy_config_export import certificate_file_name, export_caddy_config
from app_gateway.models import Application, ApplicationHost, DnsChallengeCredential
from cluster import sync
from cluster.models import ClusterSettings, Worker, WorkerStatus
from dns.models import DNSSettings
from firewall.tools import generate_firewall_script
FAKE_CERTIFICATE = '-----BEGIN CERTIFICATE-----\nfake\n-----END CERTIFICATE-----\n'
FAKE_PRIVATE_KEY = '-----BEGIN PRIVATE KEY-----\nfake\n-----END PRIVATE KEY-----\n'
class ClusterTestMixin:
"""Isolated directories for the files read and written by the cluster sync and the Caddy export."""
def setUp(self):
super().setUp()
self.temp_dir = tempfile.TemporaryDirectory()
self.addCleanup(self.temp_dir.cleanup)
self.wireguard_dir = os.path.join(self.temp_dir.name, 'wireguard')
self.dnsmasq_dir = os.path.join(self.temp_dir.name, 'dnsmasq')
self.export_dir = os.path.join(self.temp_dir.name, 'caddy_json_export')
self.certificate_export_dir = os.path.join(self.temp_dir.name, 'caddy_cert_export')
for directory in (self.wireguard_dir, self.dnsmasq_dir, self.export_dir, self.certificate_export_dir):
os.makedirs(directory)
with open(os.path.join(self.wireguard_dir, 'wg0.conf'), 'w', encoding='utf-8') as config_file:
config_file.write('[Interface]\nListenPort = 51820\n')
for target, value in (
('cluster.sync.WIREGUARD_CONFIG_DIR', self.wireguard_dir),
('cluster.sync.DNSMASQ_CONFIG_DIR', self.dnsmasq_dir),
('app_gateway.caddy_config_export.CADDY_EXPORT_DIR', self.export_dir),
('app_gateway.caddy_config_export.CERTIFICATE_EXPORT_DIR', self.certificate_export_dir),
):
patcher = patch(target, value)
patcher.start()
self.addCleanup(patcher.stop)
DNSSettings.objects.get_or_create(name='dns_settings')
ClusterSettings.objects.update_or_create(name='cluster_settings', defaults={'enabled': True})
self.worker = Worker.objects.create(name='node-a', hostname='node-a.example.test')
self.headers = {'HTTP_AUTHORIZATION': f'Bearer {self.worker.token}', 'HTTP_X_WORKER_VERSION': '12'}
def write_exported_certificate(self, hostname):
host_dir = os.path.join(self.certificate_export_dir, hostname)
os.makedirs(host_dir, exist_ok=True)
with open(os.path.join(host_dir, 'certificate.crt'), 'w', encoding='utf-8') as certificate_file:
certificate_file.write(FAKE_CERTIFICATE)
with open(os.path.join(host_dir, 'certificate.key'), 'w', encoding='utf-8') as key_file:
key_file.write(FAKE_PRIVATE_KEY)
@override_settings(CLUSTER_WORKER_MINIMUM_VERSION=12, WIREGUARD_STATUS_CACHE_ENABLED=True)
class ClusterApiTests(ClusterTestMixin, TestCase):
def test_legacy_status_endpoint_stops_old_workers_without_token(self):
response = self.client.get('/api/cluster/status/')
self.assertEqual(response.status_code, 403)
self.assertEqual(response.json()['message'], 'update_required')
self.assertEqual(response.json()['minimum_worker_version'], 12)
def test_manifest_requires_token(self):
self.assertEqual(self.client.get('/api/cluster/v2/manifest/').status_code, 403)
response = self.client.get('/api/cluster/v2/manifest/', HTTP_AUTHORIZATION='Bearer not-a-uuid')
self.assertEqual(response.status_code, 403)
def test_worker_is_refused_with_reason(self):
scenarios = (
({'HTTP_X_WORKER_VERSION': '11'}, {}, 'update_required'),
({}, {'enabled': False}, 'worker_disabled'),
({}, {'ip_lock': True, 'ip_address': '203.0.113.10'}, 'ip_lock'),
)
for header_override, worker_fields, expected_error in scenarios:
with self.subTest(expected_error=expected_error):
Worker.objects.filter(pk=self.worker.pk).update(**{'enabled': True, 'ip_lock': False, **worker_fields})
response = self.client.get('/api/cluster/v2/manifest/', **{**self.headers, **header_override})
self.assertEqual(response.status_code, 400)
self.assertEqual(response.json()['message'], expected_error)
self.worker.refresh_from_db()
self.assertEqual(self.worker.error_status, expected_error)
def test_disabled_cluster_refuses_workers(self):
ClusterSettings.objects.filter(name='cluster_settings').update(enabled=False)
response = self.client.get('/api/cluster/v2/manifest/', **self.headers)
self.assertEqual(response.status_code, 400)
self.assertEqual(response.json()['message'], 'cluster_disabled')
def test_manifest_hashes_match_sections(self):
response = self.client.get('/api/cluster/v2/manifest/', **self.headers)
self.assertEqual(response.status_code, 200)
manifest = response.json()['sections']
self.assertEqual(list(manifest), list(sync.SECTION_NAMES))
for section_name, section_hash in manifest.items():
with self.subTest(section=section_name):
section_response = self.client.get(f'/api/cluster/v2/section/{section_name}/', **self.headers)
self.assertEqual(section_response.status_code, 200)
self.assertEqual(section_response.json()['hash'], section_hash)
self.assertEqual(self.client.get('/api/cluster/v2/section/unknown/', **self.headers).status_code, 404)
def test_report_stores_worker_status(self):
report = {
'sections': {
'dns': {'hash': 'abc', 'status': 'error', 'message': 'dnsmasq failed'},
'unknown': {'hash': 'abc', 'status': 'ok'},
'settings': {'hash': 'def', 'status': 'invalid'},
},
'services': {'dnsmasq': 'running'},
'certificates': {'app.example.test': {'status': 'trusted'}},
'wireguard_status': {'wg0': {}},
}
response = self.client.post(
'/api/cluster/v2/report/', data=json.dumps(report), content_type='application/json', **self.headers
)
self.assertEqual(response.status_code, 200)
worker_status = WorkerStatus.objects.get(worker=self.worker)
self.assertEqual(list(worker_status.section_status), ['dns'])
self.assertEqual(worker_status.section_status['dns']['message'], 'dnsmasq failed')
self.assertEqual(worker_status.services_status, {'dnsmasq': 'running'})
self.assertEqual(worker_status.certificate_status, report['certificates'])
self.assertEqual(worker_status.wireguard_status, {'wg0': {}})
self.assertIsNotNone(worker_status.certificate_status_updated)
self.assertIsNotNone(worker_status.wireguard_status_updated)
class ClusterSyncTests(ClusterTestMixin, TestCase):
def report_sections(self, manifest, overrides=None):
section_status = {name: {'hash': section_hash, 'status': 'ok'} for name, section_hash in manifest.items()}
section_status.update(overrides or {})
WorkerStatus.objects.update_or_create(worker=self.worker, defaults={'section_status': section_status})
self.worker.refresh_from_db()
def test_sync_state(self):
manifest = sync.build_manifest(self.worker)
self.assertEqual(sync.get_worker_sync_state(self.worker, manifest)['state'], 'pending')
self.report_sections(manifest)
self.assertEqual(sync.get_worker_sync_state(self.worker, manifest)['state'], 'synced')
self.report_sections(manifest, {'dns': {'hash': manifest['dns'], 'status': 'error', 'message': 'failed'}})
sync_state = sync.get_worker_sync_state(self.worker, manifest)
self.assertEqual(sync_state['state'], 'error')
self.assertEqual(sync_state['sections']['dns']['message'], 'failed')
# An error reported for older content means the node still has to try the new one
self.report_sections(manifest, {'dns': {'hash': 'old', 'status': 'error', 'message': 'failed'}})
self.assertEqual(sync.get_worker_sync_state(self.worker, manifest)['state'], 'pending')
def test_wireguard_section_follows_config_files(self):
first_hash = sync.build_section('wireguard', self.worker).hash
self.assertEqual(sync.build_section('wireguard', self.worker).hash, first_hash)
with open(os.path.join(self.wireguard_dir, 'wg1.conf'), 'w', encoding='utf-8') as config_file:
config_file.write('[Interface]\nListenPort = 51821\n')
section = sync.build_section('wireguard', self.worker)
self.assertNotEqual(section.hash, first_hash)
self.assertEqual(sorted(section.files), ['wg0.conf', 'wg1.conf'])
def test_dns_section_uses_node_configuration(self):
section = sync.build_section('dns', self.worker)
main_config = section.files[sync.DNSMASQ_MAIN_CONFIG]
self.assertIn('bind-dynamic', main_config)
self.assertIn('interface=wg*', main_config)
self.assertNotIn('listen-address=0.0.0.0', main_config)
def test_firewall_section_is_stable_and_node_specific(self):
self.assertEqual(sync.build_section('firewall', self.worker).hash, sync.build_section('firewall', self.worker).hash)
node_script = generate_firewall_script(cluster_node=True)
primary_script = generate_firewall_script()
self.assertNotIn('DNS_IP', node_script)
self.assertNotIn('--dport 8000', node_script)
self.assertNotIn('generated by', node_script)
self.assertIn('DNS_IP', primary_script)
class ClusterGatewaySyncTests(ClusterTestMixin, TestCase):
def setUp(self):
super().setUp()
self.credential = DnsChallengeCredential.objects.create(
display_name='Cloudflare', provider='cloudflare', credentials={'api_token': 'x' * 40}
)
self.node_app = Application.objects.create(
display_name='Node App', upstream='http://10.0.0.20:8080', serve_on_primary=False
)
self.node_app.workers.add(self.worker)
ApplicationHost.objects.create(
application=self.node_app, hostname='node.example.test', certificate_type='dns_challenge',
dns_credential=self.credential,
)
def gateway_files(self):
return {
filename: json.loads(content)
for filename, content in sync.build_section('gateway', self.worker).files.items()
}
def test_primary_manages_certificates_of_worker_only_applications(self):
export_caddy_config(self.export_dir)
with open(os.path.join(self.export_dir, 'applications.json'), encoding='utf-8') as applications_file:
primary_applications = json.load(applications_file)
with open(os.path.join(self.export_dir, 'tls.json'), encoding='utf-8') as tls_file:
primary_tls = json.load(tls_file)
self.assertNotIn('node-app', [entry['id'] for entry in primary_applications['entries']])
host_tls = primary_tls['hosts']['node.example.test']
self.assertTrue(host_tls['certificate_only'])
self.assertTrue(host_tls['export_certificate'])
self.assertEqual(host_tls['dns_credential'], self.credential.name)
def test_workers_only_publish_exported_configuration(self):
self.assertEqual(self.gateway_files()['applications.json'], {'entries': []})
export_caddy_config(self.export_dir)
self.assertEqual([entry['id'] for entry in self.gateway_files()['applications.json']['entries']], ['node-app'])
self.node_app.upstream = 'http://10.0.0.21:8080'
self.node_app.save()
self.assertTrue(caddy_config_export.has_pending_export(self.export_dir))
self.assertEqual(self.gateway_files()['applications.json']['entries'][0]['upstream'], 'http://10.0.0.20:8080')
export_caddy_config(self.export_dir)
self.assertFalse(caddy_config_export.has_pending_export(self.export_dir))
self.assertEqual(self.gateway_files()['applications.json']['entries'][0]['upstream'], 'http://10.0.0.21:8080')
def test_worker_receives_exported_certificate_without_dns_credentials(self):
export_caddy_config(self.export_dir)
worker_tls = self.gateway_files()['tls.json']
self.assertEqual(worker_tls['dns_credentials'], {})
self.assertEqual(worker_tls['hosts']['node.example.test']['certificate_file'], '')
self.assertEqual(sync.build_section('certificates', self.worker).files, {})
# A renewed certificate reaches the worker without a new export
self.write_exported_certificate('node.example.test')
certificate_name = certificate_file_name(FAKE_CERTIFICATE)
self.assertEqual(self.gateway_files()['tls.json']['hosts']['node.example.test']['certificate_file'], certificate_name)
self.assertEqual(sync.build_section('certificates', self.worker).files, {
f'{certificate_name}.crt': FAKE_CERTIFICATE,
f'{certificate_name}.key': FAKE_PRIVATE_KEY,
})
def test_new_worker_is_pending_until_exported(self):
export_caddy_config(self.export_dir)
self.assertFalse(caddy_config_export.has_pending_export(self.export_dir))
Worker.objects.create(name='node-b')
self.assertTrue(caddy_config_export.has_pending_export(self.export_dir))
class ClusterTrafficAggregationTests(TestCase):
def test_peer_traffic_is_summed_across_servers(self):
worker = Worker.objects.create(name='node-a')
WorkerStatus.objects.create(
worker=worker, wireguard_status_updated=timezone.now(), wireguard_status={'wg0': {
'PEER': {'transfer': {'tx': 5000, 'rx': 500}, 'latest-handshakes': '2000', 'endpoints': 'node:1'},
'NODE_ONLY': {'transfer': {'tx': 7, 'rx': 3}, 'latest-handshakes': '5'},
}},
)
WireguardStatusCache.objects.create(cache_type='master', processing_time_ms=0, data={'wg0': {
'PEER': {'transfer': {'tx': 100, 'rx': 10}, 'latest-handshakes': '1000', 'endpoints': 'primary:1'},
}})
func_concatenate_cluster_wireguard_status_cache()
combined = WireguardStatusCache.objects.filter(cache_type='cluster').latest('created').data['wg0']
self.assertEqual(combined['PEER']['transfer'], {'tx': 5100, 'rx': 510})
self.assertEqual(combined['PEER']['endpoints'], 'node:1')
self.assertEqual(combined['PEER']['source_name'], 'worker_node-a')
self.assertEqual(combined['NODE_ONLY']['transfer'], {'tx': 7, 'rx': 3})
+5 -7
View File
@@ -1,12 +1,10 @@
from django.urls import path
from cluster.cluster_api import api_cluster_status, api_get_worker_config_files, api_get_worker_dnsmasq_config, \
api_worker_ping, api_submit_worker_wireguard_stats
from cluster.cluster_api import api_legacy_cluster_status, api_manifest, api_report, api_section
urlpatterns = [
path('status/', api_cluster_status, name='api_cluster_status'),
path('worker/get_config_files/', api_get_worker_config_files, name='api_get_worker_config_files'),
path('worker/get_dnsmasq_config/', api_get_worker_dnsmasq_config, name='api_get_worker_dnsmasq_config'),
path('worker/ping/', api_worker_ping, name='api_worker_ping'),
path('worker/submit_wireguard_stats/', api_submit_worker_wireguard_stats, name='api_submit_worker_wireguard_stats'),
path('status/', api_legacy_cluster_status, name='api_legacy_cluster_status'),
path('v2/manifest/', api_manifest, name='api_cluster_manifest'),
path('v2/section/<str:section_name>/', api_section, name='api_cluster_section'),
path('v2/report/', api_report, name='api_cluster_report'),
]
+2 -1
View File
@@ -1,9 +1,10 @@
from django.urls import path
from cluster.views import cluster_main, cluster_settings, worker_manage
from cluster.views import cluster_main, cluster_settings, worker_details, worker_manage
urlpatterns = [
path('', cluster_main, name='cluster_main'),
path('worker/view/', worker_details, name='worker_details'),
path('worker/manage/', worker_manage, name='worker_manage'),
path('settings/', cluster_settings, name='cluster_settings'),
]
+151 -80
View File
@@ -2,33 +2,77 @@ from django.conf import settings
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.shortcuts import get_object_or_404, redirect, render
from django.urls import reverse
from django.utils.dateparse import parse_datetime
from django.utils.translation import gettext_lazy as _
from api.models import WireguardStatusCache
from dns.functions import compress_dnsmasq_config
from app_gateway.setup_defaults import get_webadmin_hostnames
from user_manager.models import UserAcl
from .forms import WorkerForm, ClusterSettingsForm
from .models import ClusterSettings, Worker
from .models import ClusterSettings, Worker, WorkerStatus
from .sync import SECTION_NAMES, get_worker_sync_state, get_worker_warnings
def get_worker_status(worker):
try:
return worker.workerstatus
except WorkerStatus.DoesNotExist:
return None
def describe_worker(worker):
"""Attach the synchronization state, warnings and reported data used by the cluster templates."""
worker.sync_state = get_worker_sync_state(worker)
worker.unsynced_sections = [
section_name for section_name, section in worker.sync_state['sections'].items() if section['state'] != 'synced'
]
worker.error_messages = [
f"{section_name}: {section['message']}" for section_name, section in worker.sync_state['sections'].items()
if section['state'] == 'error' and section['message']
]
worker.warnings = get_worker_warnings(worker)
# Reported data of disabled, refused or offline workers is only the last known state
worker.operational = worker.enabled and not worker.error_status and worker.is_online
worker_status = get_worker_status(worker)
worker.status = worker_status
worker.services = sorted((worker_status.services_status or {}).items()) if worker_status else []
worker.runs_gateway = any(service == 'caddy' for service, state in worker.services)
worker.outdated = bool(worker_status and worker_status.worker_version != settings.CLUSTER_WORKER_CURRENT_VERSION)
return worker
@login_required
def cluster_main(request):
"""Main cluster page with workers list"""
if not UserAcl.objects.filter(user=request.user).filter(user_level__gte=50).exists():
return render(request, 'access_denied.html', {'page_title': _('Access Denied')})
cluster_settings, created = ClusterSettings.objects.get_or_create(name='cluster_settings')
page_title = _('Cluster')
workers = Worker.objects.all().order_by('name')
master_cache_times = ", ".join([str(c.processing_time_ms) for c in WireguardStatusCache.objects.filter(cache_type='master').order_by('-created')])
cluster_cache_times = ", ".join([str(c.processing_time_ms) for c in WireguardStatusCache.objects.filter(cache_type='cluster').order_by('-created')])
cluster_settings, cluster_settings_created = ClusterSettings.objects.get_or_create(name='cluster_settings')
workers = [describe_worker(worker) for worker in Worker.objects.all().select_related('workerstatus').order_by('name')]
enabled_workers = [worker for worker in workers if worker.enabled]
master_cache_times = ", ".join(
str(cache.processing_time_ms) for cache in WireguardStatusCache.objects.filter(cache_type='master').order_by('-created')
)
cluster_cache_times = ", ".join(
str(cache.processing_time_ms) for cache in WireguardStatusCache.objects.filter(cache_type='cluster').order_by('-created')
)
context = {
'page_title': page_title,
'page_title': _('Cluster'),
'workers': workers,
'current_worker_version': settings.CLUSTER_WORKER_CURRENT_VERSION,
'cluster_settings': cluster_settings,
'current_worker_version': settings.CLUSTER_WORKER_CURRENT_VERSION,
'summary': {
'enabled': len(enabled_workers),
'online': sum(1 for worker in enabled_workers if worker.operational),
'synced': sum(1 for worker in enabled_workers if worker.operational and worker.sync_state['state'] == 'synced'),
'attention': sum(
1 for worker in enabled_workers
if worker.sync_state['state'] == 'error' or worker.error_status or worker.warnings['stopped_services']
or worker.warnings['certificate_issues'] or not worker.is_online
),
},
'cache_refresh_interval': settings.WIREGUARD_STATUS_CACHE_REFRESH_INTERVAL,
'cache_enabled': settings.WIREGUARD_STATUS_CACHE_ENABLED,
'cache_web_load_previous_count': settings.WIREGUARD_STATUS_CACHE_WEB_LOAD_PREVIOUS_COUNT,
@@ -39,119 +83,146 @@ def cluster_main(request):
@login_required
def worker_manage(request):
"""Add/Edit worker view"""
def worker_details(request):
if not UserAcl.objects.filter(user=request.user).filter(user_level__gte=50).exists():
return render(request, 'access_denied.html', {'page_title': _('Access Denied')})
worker = describe_worker(get_object_or_404(Worker.objects.select_related('workerstatus'), uuid=request.GET.get('uuid')))
cluster_settings, cluster_settings_created = ClusterSettings.objects.get_or_create(name='cluster_settings')
sections = []
for section_name in SECTION_NAMES:
section_state = worker.sync_state['sections'].get(section_name, {})
reported = (worker.status.section_status or {}).get(section_name, {}) if worker.status else {}
sections.append({
'name': section_name,
'state': section_state.get('state', 'pending'),
'message': section_state.get('message', ''),
'updated': parse_datetime(reported.get('updated', '')) if reported.get('updated') else None,
})
certificates = []
if worker.status:
for hostname, certificate in sorted((worker.status.certificate_status or {}).items()):
certificates.append({
'hostname': hostname,
'status': certificate.get('status', ''),
'issuer': certificate.get('issuer', ''),
'not_after': parse_datetime(certificate['not_after']) if certificate.get('not_after') else None,
})
webadmin_hostnames = get_webadmin_hostnames()
primary_address = f"https://{webadmin_hostnames[0] if webadmin_hostnames else request.get_host()}"
context = {
'page_title': worker.name,
'worker': worker,
'cluster_settings': cluster_settings,
'current_worker_version': settings.CLUSTER_WORKER_CURRENT_VERSION,
'sections': sections,
'certificates': certificates,
'primary_address': primary_address,
'time_zone': settings.TIME_ZONE,
}
return render(request, 'cluster/worker_details.html', context)
@login_required
def worker_manage(request):
if not UserAcl.objects.filter(user=request.user).filter(user_level__gte=50).exists():
return render(request, 'access_denied.html', {'page_title': _('Access Denied')})
worker = None
if 'uuid' in request.GET:
worker = get_object_or_404(Worker, uuid=request.GET['uuid'])
form = WorkerForm(instance=worker)
page_title = _('Edit Worker: ') + worker.name
if request.GET.get('action') == 'delete':
worker_name = worker.name
if request.GET.get('confirmation') == 'delete':
worker.delete()
messages.success(request, _('Worker deleted|Worker deleted: ') + worker_name)
return redirect('/cluster/')
else:
messages.warning(request, _('Worker not deleted|Invalid confirmation.'))
return redirect('/cluster/')
return redirect(reverse('cluster_main'))
else:
form = WorkerForm()
page_title = _('Add Worker')
if request.method == 'POST':
if worker:
form = WorkerForm(request.POST, instance=worker)
form = WorkerForm(request.POST or None, instance=worker)
if form.is_valid():
is_new_worker = worker is None
worker = form.save()
if is_new_worker:
messages.success(request, _('Worker created|Worker created: ') + worker.name)
else:
form = WorkerForm(request.POST)
if form.is_valid():
worker = form.save()
if worker.pk:
messages.success(request, _('Worker updated|Worker updated: ') + worker.name)
else:
messages.success(request, _('Worker created|Worker created: ') + worker.name)
return redirect('/cluster/')
messages.success(request, _('Worker updated|Worker updated: ') + worker.name)
return redirect(reverse('worker_details') + f'?uuid={worker.uuid}')
form_description = {
'size': 'col-lg-6',
'content': _('''
<h5>Worker Configuration</h5>
<p>Configure a cluster worker node that will synchronize with this primary instance.</p>
<h5>Name</h5>
<p>A unique name to identify this worker.</p>
<h5>IP Address</h5>
<p>The IP address of the worker node. Leave empty if IP lock is disabled.</p>
<h5>IP Lock</h5>
<p>When enabled, the worker can only connect from the specified IP address.</p>
<h5>Location Information</h5>
<p>Optional location details for this worker (country, city, hostname).</p>
<h5>Worker</h5>
<p>A worker is a cluster node that replicates WireGuard, firewall and DNS from this primary server and, optionally, publishes App Gateway applications.</p>
<h5>Name and location</h5>
<p>Used to identify the worker. The <strong>hostname</strong> is the address clients use to connect to this worker: it is written as the endpoint of the peer configurations downloaded for this server.</p>
<h5>Token</h5>
<p>Secret used by the worker to authenticate with this primary server. Changing it disconnects the worker until its configuration is updated.</p>
<h5>IP lock</h5>
<p>When enabled, the worker is only accepted from the IP address below. Without it, the address is updated automatically from the last connection.</p>
<h5>Enabled</h5>
<p>Disabled workers are refused and shut down their WireGuard interfaces until enabled again.</p>
''')
}
context = {
'page_title': page_title,
'form': form,
'worker': worker,
'page_title': page_title,
'form': form,
'worker': worker,
'instance': worker,
'form_description': form_description
'form_description': form_description,
}
return render(request, 'generic_form.html', context)
@login_required
def cluster_settings(request):
"""Cluster settings configuration"""
if not UserAcl.objects.filter(user=request.user).filter(user_level__gte=50).exists():
return render(request, 'access_denied.html', {'page_title': _('Access Denied')})
cluster_settings, created = ClusterSettings.objects.get_or_create(name='cluster_settings')
page_title = _('Cluster Settings')
if request.method == 'POST':
form = ClusterSettingsForm(request.POST, instance=cluster_settings)
if form.is_valid():
form.save()
messages.success(request, _('Cluster settings updated successfully.'))
if cluster_settings.enabled:
if cluster_settings.config_version == 0:
cluster_settings.config_version += 1
cluster_settings.save()
compress_dnsmasq_config()
return redirect('/cluster/')
else:
form = ClusterSettingsForm(instance=cluster_settings)
cluster_settings, cluster_settings_created = ClusterSettings.objects.get_or_create(name='cluster_settings')
form = ClusterSettingsForm(request.POST or None, instance=cluster_settings)
if form.is_valid():
form.save()
messages.success(request, _('Cluster settings updated successfully.'))
return redirect(reverse('cluster_main'))
form_description = {
'size': 'col-lg-6',
'content': _('''
<h5>Cluster Enabled</h5>
<p>When disabled, workers are refused and shut down their WireGuard interfaces, and App Gateway applications are only published by this primary server. Requires the WireGuard status cache.</p>
<h5>Primary Enable WireGuard</h5>
<p>Lists this primary server as an option when downloading peer configurations and VPN invites.</p>
<h5>Cluster Mode</h5>
<p>Configure how the cluster operates and synchronizes configurations between nodes.</p>
<h5>Sync Intervals</h5>
<p>Configure how frequently statistics and cache data are synchronized between cluster nodes.</p>
<h5>Restart Mode</h5>
<p>Choose whether WireGuard services should be automatically restarted when configurations change, or if manual intervention is required.</p>
<p><strong>Mirror</strong>: every worker receives the same WireGuard instances, peers, firewall and DNS configuration. Peers can connect to any server.</p>
<h5>Worker Display</h5>
<p>Select how workers should be identified in the interface - by name, server address, location, or a combination.</p>
<p>How workers are named in server selections (peer configuration downloads and VPN invites).</p>
''')
}
context = {
'page_title': page_title,
'page_title': _('Cluster Settings'),
'form': form,
'cluster_settings': cluster_settings,
'instance': cluster_settings,
'form_description': form_description
'form_description': form_description,
}
return render(request, 'generic_form.html', context)
+54 -6
View File
@@ -1,4 +1,3 @@
version: '3'
services:
cluster-node:
container_name: cluster-node
@@ -8,16 +7,19 @@ services:
dockerfile: Dockerfile-cluster_node
environment:
- MASTER_SERVER_ADDRESS=${MASTER_SERVER_ADDRESS}
- DEBUG_MODE=${DEBUG_MODE}
- COMPOSE_VERSION=c1b
- COMPOSE_VERSION=c2a
- TZ=${TIMEZONE}
- TOKEN=${TOKEN}
volumes:
- cluster_node_wireguard:/etc/wireguard
- cluster_node_dnsmasq_conf:/etc/dnsmasq/
- cluster_node_caddy_json:/caddy_json_export/
- cluster_node_caddy_certificates:/caddy_certificates/
ports:
# Ports for WireGuard instances.
- "51820-51839:51820-51839/udp"
# Different host ports, so the node can run on the same host as the development primary server
- "52820-52839:51820-51839/udp"
- "9080:80"
- "9443:443"
cap_add:
- NET_ADMIN
- SYS_MODULE
@@ -28,11 +30,57 @@ services:
cluster-node-dns:
container_name: cluster-node-dns
restart: unless-stopped
image: eduardosilva/wireguard_webadmin_dns:latest
build:
context: containers/dnsmasq
dockerfile: Dockerfile-dnsmasq
network_mode: "service:cluster-node"
environment:
- TZ=${TIMEZONE}
volumes:
- cluster_node_dnsmasq_conf:/etc/dnsmasq/
depends_on:
- cluster-node
cluster-node-caddy:
container_name: cluster-node-caddy
restart: unless-stopped
build:
context: containers/caddy
dockerfile: Dockerfile-caddy
network_mode: "service:cluster-node"
environment:
- TZ=${TIMEZONE}
- CLUSTER_NODE=true
- AUTH_GATEWAY_INTERNAL_URL=http://cluster-node-auth-gateway:9091
volumes:
- cluster_node_caddy_json:/caddy_json_export/
- cluster_node_caddy_certificates:/caddy_certificates/:ro
- cluster_node_caddy_data:/data
- cluster_node_caddy_config:/config
depends_on:
- cluster-node
- cluster-node-auth-gateway
cluster-node-auth-gateway:
container_name: cluster-node-auth-gateway
restart: unless-stopped
build:
context: containers/auth-gateway
dockerfile: Dockerfile-auth-gateway
environment:
- TZ=${TIMEZONE}
- AUTH_GATEWAY_CONFIG_DIR=/caddy_json_export
- AUTH_GATEWAY_DATABASE_PATH=/data/auth-gateway.sqlite3
- AUTH_GATEWAY_SECURE_COOKIES=false
volumes:
- cluster_node_caddy_json:/caddy_json_export/
- cluster_node_auth_gateway_data:/data
volumes:
cluster_node_wireguard:
cluster_node_dnsmasq_conf:
cluster_node_caddy_json:
cluster_node_caddy_certificates:
cluster_node_caddy_data:
cluster_node_caddy_config:
cluster_node_auth_gateway_data:
+48
View File
@@ -0,0 +1,48 @@
# App Gateway for cluster nodes. Use together with cluster_node.yml:
# docker compose -f cluster_node.yml -f cluster_node-gateway.yml up -d
services:
cluster-node:
volumes:
- cluster_node_caddy_json:/caddy_json_export/
- cluster_node_caddy_certificates:/caddy_certificates/
ports:
# Caddy shares the network of cluster-node, so its ports are published here
- "80:80"
- "443:443"
cluster-node-caddy:
container_name: cluster-node-caddy
restart: unless-stopped
image: eduardosilva/wireguard_webadmin_caddy:latest
network_mode: "service:cluster-node"
environment:
- TZ=${TIMEZONE}
- CLUSTER_NODE=true
- AUTH_GATEWAY_INTERNAL_URL=http://cluster-node-auth-gateway:9091
volumes:
- cluster_node_caddy_json:/caddy_json_export/
- cluster_node_caddy_certificates:/caddy_certificates/:ro
- cluster_node_caddy_data:/data
- cluster_node_caddy_config:/config
depends_on:
- cluster-node
- cluster-node-auth-gateway
cluster-node-auth-gateway:
container_name: cluster-node-auth-gateway
restart: unless-stopped
image: eduardosilva/wireguard_webadmin_auth_gateway:latest
environment:
- TZ=${TIMEZONE}
- AUTH_GATEWAY_CONFIG_DIR=/caddy_json_export
- AUTH_GATEWAY_DATABASE_PATH=/data/auth-gateway.sqlite3
volumes:
- cluster_node_caddy_json:/caddy_json_export/
- cluster_node_auth_gateway_data:/data
volumes:
cluster_node_caddy_json:
cluster_node_caddy_certificates:
cluster_node_caddy_data:
cluster_node_caddy_config:
cluster_node_auth_gateway_data:
+6 -3
View File
@@ -1,4 +1,3 @@
version: '3'
services:
cluster-node:
container_name: cluster-node
@@ -6,8 +5,7 @@ services:
image: eduardosilva/wireguard-webadmin-cluster-node:latest
environment:
- MASTER_SERVER_ADDRESS=${MASTER_SERVER_ADDRESS}
- DEBUG_MODE=${DEBUG_MODE}
- COMPOSE_VERSION=c1b
- COMPOSE_VERSION=c2a
- TZ=${TIMEZONE}
- TOKEN=${TOKEN}
volumes:
@@ -27,10 +25,15 @@ services:
container_name: cluster-node-dns
restart: unless-stopped
image: eduardosilva/wireguard_webadmin_dns:latest
# Shares the network of cluster-node, so DNS is answered directly on the WireGuard interfaces
network_mode: "service:cluster-node"
environment:
- TZ=${TIMEZONE}
volumes:
- cluster_node_dnsmasq_conf:/etc/dnsmasq/
depends_on:
- cluster-node
volumes:
cluster_node_wireguard:
cluster_node_dnsmasq_conf:
@@ -0,0 +1,40 @@
{
"hosts": {
"dashboard.example.test": {
"certificate_type": "http_challenge",
"certificate_fallback": true,
"certificate_generation": 0
},
"hypervisor.example.test": {
"certificate_type": "http_challenge",
"certificate_fallback": true,
"certificate_generation": 0
},
"media.example.test": {
"certificate_type": "http_challenge",
"certificate_fallback": true,
"certificate_generation": 0
},
"media2.example.test": {
"certificate_type": "http_challenge",
"certificate_fallback": true,
"certificate_generation": 0
},
"sso.example.test": {
"certificate_type": "http_challenge",
"certificate_fallback": true,
"certificate_generation": 0
},
"webadmin.example.test": {
"certificate_type": "http_challenge",
"certificate_fallback": true,
"certificate_generation": 0
},
"webadmin2.example.test": {
"certificate_type": "http_challenge",
"certificate_fallback": true,
"certificate_generation": 0
}
},
"dns_credentials": {}
}
+50 -10
View File
@@ -7,16 +7,20 @@ PYTHON="python3"
JSON_DIR="${JSON_DIR:-/caddy_json_export}"
CADDYFILE_PATH="${CADDYFILE_PATH:-/etc/caddy/Caddyfile}"
CADDY_PID=""
CLUSTER_NODE="${CLUSTER_NODE:-false}"
echo "==> Generating wireguard_webadmin.json..."
"$PYTHON" "${SCRIPT_DIR}/export_wireguard_webadmin_config.py"
# The WireGuard WebAdmin interface is only published by the primary server
if [ "$CLUSTER_NODE" != "true" ]; then
echo "==> Generating wireguard_webadmin.json..."
"$PYTHON" "${SCRIPT_DIR}/export_wireguard_webadmin_config.py"
fi
generate_caddyfile() {
"$PYTHON" "${SCRIPT_DIR}/process_config.py"
"$PYTHON" "${SCRIPT_DIR}/process_config.py" >/dev/null
# Validation output is not printed because provider errors may include credentials
if ! caddy validate --config "$CADDYFILE_PATH" --adapter caddyfile >/dev/null 2>&1; then
echo "Warning: generated Caddyfile is invalid, regenerating with DNS challenge disabled. Check the DNS credentials."
DISABLE_DNS_CHALLENGE=1 "$PYTHON" "${SCRIPT_DIR}/process_config.py"
DISABLE_DNS_CHALLENGE=1 "$PYTHON" "${SCRIPT_DIR}/process_config.py" >/dev/null
fi
}
@@ -42,27 +46,63 @@ stop_caddy() {
echo "==> Processing config files..."
generate_caddyfile
sync_tls_state >/dev/null
# Nothing is cached yet, so hosts left on a fallback certificate get a new ACME attempt
"$PYTHON" "${SCRIPT_DIR}/process_config.py" --reset-fallback-certificates
echo "==> Starting Caddy..."
start_caddy
if [ "$CLUSTER_NODE" != "true" ]; then
# Certificates used by cluster workers are exported as soon as they are issued or renewed
(
while true; do
sleep 60
"$PYTHON" "${SCRIPT_DIR}/process_config.py" --export-certificates || true
done
) &
fi
sleep 2
caddyfile_checksum() {
sha256sum "$CADDYFILE_PATH" 2>/dev/null | cut -d ' ' -f 1
}
APPLIED_CADDYFILE="$(caddyfile_checksum)"
echo "==> Watching ${JSON_DIR} for config changes..."
while true; do
inotifywait -qq -e close_write,moved_to,create "${JSON_DIR}/" --include '.*\.json$' 2>/dev/null || true
# The timeout also picks up changes written before the watch started or missed while reloading
inotifywait -qq -t 60 -e close_write,moved_to,create "${JSON_DIR}/" --include '.*\.json$' 2>/dev/null || true
sleep 1
echo "==> Config change detected, reprocessing..."
# The container that owns this network namespace was restarted: exit so Docker restarts Caddy and it
# joins the new namespace (otherwise it keeps serving from a namespace without published ports)
if [ -f "${JSON_DIR}/.netns_owner" ] && [ "$(cat "${JSON_DIR}/.netns_owner")" != "$(readlink /proc/self/ns/net)" ]; then
echo "Network namespace owner changed, restarting..."
stop_caddy
exit 1
fi
generate_caddyfile
if [ "$(sync_tls_state)" = "restart" ]; then
if ! kill -0 "$CADDY_PID" 2>/dev/null; then
echo "==> Caddy is not running, starting it again..."
sync_tls_state >/dev/null
start_caddy
APPLIED_CADDYFILE="$(caddyfile_checksum)"
elif [ "$(sync_tls_state)" = "restart" ]; then
# A reload keeps previously issued certificates cached in memory
echo "==> Certificate settings changed, restarting Caddy..."
stop_caddy
start_caddy
else
echo "==> Reloading Caddy..."
caddy reload --config "$CADDYFILE_PATH" --adapter caddyfile 2>/dev/null || echo "Warning: Caddy reload failed, will retry on next change."
APPLIED_CADDYFILE="$(caddyfile_checksum)"
elif [ "$(caddyfile_checksum)" != "$APPLIED_CADDYFILE" ]; then
echo "==> Config change detected, reloading Caddy..."
if caddy reload --config "$CADDYFILE_PATH" --adapter caddyfile 2>/dev/null; then
APPLIED_CADDYFILE="$(caddyfile_checksum)"
else
echo "Warning: Caddy reload failed, will retry on next change."
fi
fi
done
+171 -17
View File
@@ -36,6 +36,10 @@ TLS_STATE_PATH = os.path.join(CADDY_DATA_DIR, "wireguard_webadmin_tls_state.json
TLS_STATE_PENDING_PATH = f"{CADDYFILE_PATH}.tls_state.json"
# Set by entrypoint.sh when the generated Caddyfile fails validation
DISABLE_DNS_CHALLENGE = os.environ.get("DISABLE_DNS_CHALLENGE", "") == "1"
# Cluster nodes never request ACME certificates: they use the ones exported by the primary server
CLUSTER_NODE = os.environ.get("CLUSTER_NODE", "").lower() == "true"
NODE_CERTIFICATES_DIR = os.environ.get("NODE_CERTIFICATES_DIR", "/caddy_certificates")
CERTIFICATE_EXPORT_DIR = os.environ.get("CERTIFICATE_EXPORT_DIR", "/caddy_cert_export")
AUTH_IDENTITY_HEADERS = (
"X-Auth-User",
"X-Auth-Email",
@@ -92,12 +96,30 @@ def _safe_identifier(val: str) -> str:
return re.sub(r"[^A-Za-z0-9_]", "", str(val))
def get_node_host_tls(host_tls):
"""
TLS settings of a host on a cluster node: the certificate exported by the primary server, the internal CA
as fallback, or None when there is no certificate yet and the fallback is disabled.
"""
certificate_type = host_tls.get("certificate_type", DEFAULT_CERTIFICATE_TYPE)
certificate_fallback = host_tls.get("certificate_fallback", True) is not False
certificate_file = re.sub(r"[^a-f0-9]", "", str(host_tls.get("certificate_file", "")))
if certificate_type in ACME_CERTIFICATE_TYPES and certificate_file:
return "certificate_file", certificate_file, certificate_fallback
if certificate_type == "self_signed" or certificate_fallback:
return "self_signed", None, True
return None
def get_host_tls(host, tls_data):
"""
Return (certificate_type, dns_credential_name, certificate_fallback) for a host.
Hosts not listed use the HTTP challenge with fallback.
Hosts not listed use the HTTP challenge with fallback. On cluster nodes the second item is the
certificate file name, and None is returned for hosts that must not be served.
"""
host_tls = tls_data.get("hosts", {}).get(host.lower(), {})
if CLUSTER_NODE:
return get_node_host_tls(host_tls)
certificate_type = host_tls.get("certificate_type", DEFAULT_CERTIFICATE_TYPE)
if certificate_type not in CERTIFICATE_TYPES:
certificate_type = DEFAULT_CERTIFICATE_TYPE
@@ -117,15 +139,29 @@ def group_hosts_by_tls(hosts, tls_data):
"""Hosts sharing the same certificate settings are served by the same site block."""
groups = {}
for host in hosts:
groups.setdefault(get_host_tls(host, tls_data), []).append(host)
host_tls = get_host_tls(host, tls_data)
if host_tls is not None:
groups.setdefault(host_tls, []).append(host)
return groups.items()
def get_certificate_only_hosts(apps, tls_data):
"""Hosts the primary server only manages certificates for, because their application is served by workers."""
if CLUSTER_NODE:
return []
served_hosts = {host.lower() for app in apps for host in app.get("hosts", [])}
return [
host for host, host_tls in tls_data.get("hosts", {}).items()
if host_tls.get("certificate_only") and host not in served_hosts
]
def build_caddyfile(apps, auth_policies, routes, tls_data=None):
tls_data = tls_data or {}
policies = auth_policies.get("policies", {}) if auth_policies else {}
route_entries = routes.get("entries", {}) if routes else {}
lines = []
# Caddy refuses an empty Caddyfile, which happens on cluster nodes without applications
lines = ["# Generated by process_config.py. Do not edit.", ""]
def get_policy_type(policy_name):
if policy_name and policy_name in policies:
@@ -144,6 +180,12 @@ def build_caddyfile(apps, auth_policies, routes, tls_data=None):
lines.append("")
return
if certificate_type == "certificate_file":
certificate_path = os.path.join(NODE_CERTIFICATES_DIR, dns_credential)
lines.append(f" tls {certificate_path}.crt {certificate_path}.key")
lines.append("")
return
lines.append(" tls {")
if certificate_type == "dns_challenge":
dns_credential_data = tls_data["dns_credentials"][dns_credential]
@@ -299,6 +341,15 @@ def build_caddyfile(apps, auth_policies, routes, tls_data=None):
for (certificate_type, dns_credential, certificate_fallback), group_hosts in group_hosts_by_tls(hosts, tls_data):
emit_site_block(app, group_hosts, certificate_type, dns_credential, certificate_fallback)
certificate_only_hosts = get_certificate_only_hosts(apps, tls_data)
for (certificate_type, dns_credential, certificate_fallback), group_hosts in group_hosts_by_tls(certificate_only_hosts, tls_data):
# Keeps the certificate managed for cluster workers; the application is not published here
lines.append(f"{', '.join(_safe_value(host) for host in group_hosts)} {{")
emit_tls(certificate_type, dns_credential, certificate_fallback)
lines.append(" respond 404")
lines.append("}")
lines.append("")
return "\n".join(lines)
@@ -306,19 +357,21 @@ def build_tls_state(apps, tls_data):
"""Map each host to a fingerprint of the certificate settings used in the generated Caddyfile."""
tls_data = tls_data or {}
tls_state = {}
for app in apps:
if not app.get("hosts") or not app.get("upstream"):
hosts = [host for app in apps if app.get("upstream") for host in app.get("hosts", [])]
hosts += get_certificate_only_hosts(apps, tls_data)
for host in hosts:
host_tls = get_host_tls(host, tls_data)
if host_tls is None:
continue
for host in app["hosts"]:
certificate_type, dns_credential, certificate_fallback = get_host_tls(host, tls_data)
credential_hash = ""
if dns_credential:
credential_data = json.dumps(tls_data["dns_credentials"][dns_credential], sort_keys=True)
credential_hash = hashlib.sha256(credential_data.encode()).hexdigest()
certificate_generation = tls_data.get("hosts", {}).get(host.lower(), {}).get("certificate_generation", 0)
tls_state[host.lower()] = [
certificate_type, dns_credential or "", credential_hash, certificate_fallback, certificate_generation
]
certificate_type, certificate_reference, certificate_fallback = host_tls
credential_hash = ""
if certificate_type == "dns_challenge":
credential_data = json.dumps(tls_data["dns_credentials"][certificate_reference], sort_keys=True)
credential_hash = hashlib.sha256(credential_data.encode()).hexdigest()
certificate_generation = tls_data.get("hosts", {}).get(host.lower(), {}).get("certificate_generation", 0)
tls_state[host.lower()] = [
certificate_type, certificate_reference or "", credential_hash, certificate_fallback, certificate_generation
]
return tls_state
@@ -363,9 +416,22 @@ def sync_tls_state():
restart_required = False
if applied_state is not None:
for host, host_state in pending_state.items():
if applied_state.get(host) in (None, host_state):
previous_host_state = applied_state.get(host)
if previous_host_state == host_state or "/" in host or host in ("", ".", ".."):
continue
if "/" in host or host in ("", ".", ".."):
# A new host normally has nothing cached, unless a fallback certificate was left by an earlier use
# of the same hostname (e.g. removed from one application and added to another)
if previous_host_state is None and not os.path.isdir(
os.path.join(CERTIFICATES_DIR, INTERNAL_ISSUER_DIR_NAME, host)
):
continue
if CLUSTER_NODE:
# Certificates received from the primary server replace cached ones only after a restart
if host_state[0] == "certificate_file":
fallback_certificate_dir = os.path.join(CERTIFICATES_DIR, INTERNAL_ISSUER_DIR_NAME, host)
if os.path.isdir(fallback_certificate_dir):
shutil.rmtree(fallback_certificate_dir)
restart_required = True
continue
if host_state[0] not in ACME_CERTIFICATE_TYPES:
# A cached ACME certificate would keep being served instead of the self-signed one
@@ -386,6 +452,90 @@ def sync_tls_state():
print("restart")
def reset_fallback_certificates():
"""
Remove fallback certificates of ACME hosts before Caddy starts, so the ACME certificate is requested again
on every start instead of only on the next renewal. If the request fails, the fallback is issued again.
"""
pending_state = _read_json(TLS_STATE_PENDING_PATH) or {}
for host, host_state in pending_state.items():
if host_state[0] not in ACME_CERTIFICATE_TYPES or "/" in host or host in ("", ".", ".."):
continue
fallback_certificate_dir = os.path.join(CERTIFICATES_DIR, INTERNAL_ISSUER_DIR_NAME, host)
if os.path.isdir(fallback_certificate_dir):
shutil.rmtree(fallback_certificate_dir)
print(f"Removed fallback certificate for {host}", file=sys.stderr)
def find_acme_certificate(host):
"""Return (certificate_path, key_path) of the newest ACME certificate stored by Caddy for a host."""
candidates = []
if os.path.isdir(CERTIFICATES_DIR):
for issuer_dir_name in os.listdir(CERTIFICATES_DIR):
if issuer_dir_name == INTERNAL_ISSUER_DIR_NAME:
continue
certificate_path = os.path.join(CERTIFICATES_DIR, issuer_dir_name, host, f"{host}.crt")
key_path = os.path.join(CERTIFICATES_DIR, issuer_dir_name, host, f"{host}.key")
if os.path.isfile(certificate_path) and os.path.isfile(key_path):
candidates.append((os.path.getmtime(certificate_path), certificate_path, key_path))
if not candidates:
return None, None
newest = max(candidates)
return newest[1], newest[2]
def _copy_if_changed(source_path, destination_path):
with open(source_path, "r", encoding="utf-8") as source_file:
content = source_file.read()
if _read_text(destination_path) == content:
return False
temp_path = f"{destination_path}.tmp"
with open(temp_path, "w", encoding="utf-8") as destination_file:
destination_file.write(content)
os.chmod(temp_path, 0o600)
os.replace(temp_path, destination_path)
return True
def _read_text(filepath):
try:
with open(filepath, "r", encoding="utf-8") as text_file:
return text_file.read()
except OSError:
return None
def export_certificates():
"""
Copy the ACME certificates of hosts served by cluster workers to CERTIFICATE_EXPORT_DIR, where Django reads
them to distribute to the workers. Fallback certificates from the internal CA are never exported.
"""
if CLUSTER_NODE or not os.path.isdir(CERTIFICATE_EXPORT_DIR):
return
tls_data = load_json("tls.json") or {}
export_hosts = {
host for host, host_tls in tls_data.get("hosts", {}).items()
if host_tls.get("export_certificate") and "/" not in host and host not in ("", ".", "..")
}
for host in sorted(export_hosts):
certificate_path, key_path = find_acme_certificate(host)
if not certificate_path:
continue
host_export_dir = os.path.join(CERTIFICATE_EXPORT_DIR, host)
os.makedirs(host_export_dir, exist_ok=True)
key_changed = _copy_if_changed(key_path, os.path.join(host_export_dir, "certificate.key"))
certificate_changed = _copy_if_changed(certificate_path, os.path.join(host_export_dir, "certificate.crt"))
if certificate_changed or key_changed:
print(f"Exported certificate for {host}")
for host in os.listdir(CERTIFICATE_EXPORT_DIR):
host_export_dir = os.path.join(CERTIFICATE_EXPORT_DIR, host)
if host not in export_hosts and os.path.isdir(host_export_dir):
shutil.rmtree(host_export_dir)
print(f"Removed exported certificate for {host}")
def main():
apps = collect_all_applications()
auth_policies = load_json("auth_policies.json")
@@ -403,5 +553,9 @@ def main():
if __name__ == "__main__":
if "--sync-tls-state" in sys.argv:
sync_tls_state()
elif "--export-certificates" in sys.argv:
export_certificates()
elif "--reset-fallback-certificates" in sys.argv:
reset_fallback_certificates()
else:
main()
@@ -12,6 +12,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
inetutils-traceroute \
procps \
curl \
openssl \
nano \
&& rm -rf /var/lib/apt/lists/*
+394 -312
View File
@@ -1,350 +1,432 @@
import glob
import json
import logging
import os
import re
import socket
import ssl
import subprocess
import tempfile
import time
from typing import Dict, Any
from datetime import datetime, timezone
import requests
# Configure logging
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s'
)
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
# Constants
MASTER_SERVER_ADDRESS = os.environ.get('MASTER_SERVER_ADDRESS')
TOKEN = os.environ.get('TOKEN')
MASTER_SERVER_ADDRESS = os.environ.get('MASTER_SERVER_ADDRESS', '')
TOKEN = os.environ.get('TOKEN', '')
WORKER_VERSION = 12
REQUEST_TIMEOUT = 30
MANIFEST_INTERVAL = 60
DEFAULT_STATS_INTERVAL = 60
WIREGUARD_DIR = '/etc/wireguard'
FIREWALL_SCRIPT = os.path.join(WIREGUARD_DIR, 'wg-firewall.sh')
DNS_DIR = '/etc/dnsmasq'
WORKER_VERSION = 11
REQUEST_TIMEOUT = 20
class ClusterWorker:
def __init__(self):
self.config_version = 0
self.dns_version = self.get_local_dns_version()
self.should_run = True
self.session = requests.Session()
if not MASTER_SERVER_ADDRESS or not TOKEN:
logger.error("MASTER_SERVER_ADDRESS or TOKEN not set")
self.should_run = False
self.base_url = f"https://{MASTER_SERVER_ADDRESS}/api/cluster"
# Validate URL scheme
if not MASTER_SERVER_ADDRESS.startswith(('http://', 'https://')):
self.base_url = f"https://{MASTER_SERVER_ADDRESS}/api/cluster"
else:
self.base_url = f"{MASTER_SERVER_ADDRESS}/api/cluster"
DNS_MAIN_CONFIG = 'wireguard_webadmin_dns.conf'
# Only mounted when the node runs the App Gateway (cluster_node-gateway.yml)
GATEWAY_DIR = '/caddy_json_export'
CERTIFICATES_DIR = '/caddy_certificates'
GATEWAY_FILES = ('applications.json', 'auth_policies.json', 'routes.json', 'tls.json')
CERTIFICATE_FILE_PATTERN = re.compile(r'^[a-f0-9]{64}\.(crt|key)$')
CADDY_ADMIN_URL = 'http://127.0.0.1:2019/config/'
AUTH_GATEWAY_URL = 'http://cluster-node-auth-gateway:9091/'
INTERNAL_CA_NAME = 'Caddy Local Authority'
FIREWALL_CHAINS = (
('nat', 'WGWADM_POSTROUTING'),
('nat', 'WGWADM_PREROUTING'),
('filter', 'WGWADM_FORWARD'),
('filter', 'WGWADM_ROUTE_POLICY'),
)
def func_process_wireguard_status(self) -> Dict[str, Any]:
command = "wg show all dump"
process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
stdout, stderr = process.communicate()
if process.returncode != 0:
return {"message": stderr, "status": "error"}
data: Dict[str, Dict[str, Dict[str, Any]]] = {}
# wg dump format is tab-separated.
# There are two kinds of lines:
# - Interface line: interface \t private_key \t public_key \t listen_port \t fwmark
# - Peer line: interface \t peer_public_key \t preshared_key \t endpoint \t allowed_ips \t latest_handshake \t transfer_rx \t transfer_tx \t persistent_keepalive
for line in stdout.strip().split("\n"):
parts = line.split("\t")
if len(parts) < 5:
continue
interface = parts[0]
# Peer lines are expected to have at least 9 fields
if len(parts) < 9:
# interface line, ignore
continue
peer_public_key = parts[1]
endpoint = parts[3]
allowed_ips_raw = parts[4]
latest_handshake = parts[5]
transfer_rx = parts[6]
transfer_tx = parts[7]
if interface not in data:
data[interface] = {}
data[interface][peer_public_key] = {
"allowed-ips": [ip for ip in allowed_ips_raw.split(",") if ip] if allowed_ips_raw else [],
"latest-handshakes": latest_handshake or "",
"transfer": {"tx": int(transfer_tx or 0), "rx": int(transfer_rx or 0)},
"endpoints": endpoint or "",
}
return data
class SectionError(Exception):
pass
def cleanup_wireguard(self):
logger.info("Cleaning up WireGuard configuration...")
# Stop all wireguard interfaces
try:
files = glob.glob(f"{WIREGUARD_DIR}/*.conf")
for f in files:
interface = os.path.basename(f).replace('.conf', '')
subprocess.run(['wg-quick', 'down', interface], capture_output=True)
except Exception as e:
logger.error(f"Error stopping wireguard interfaces: {e}")
def run_command(command, check=True):
result = subprocess.run(command, capture_output=True, text=True)
if check and result.returncode != 0:
raise SectionError(f"{' '.join(command)}: {(result.stderr or result.stdout).strip()}")
return result
# Remove files
try:
for f in glob.glob(f"{WIREGUARD_DIR}/*"):
os.remove(f)
except Exception as e:
logger.error(f"Error cleaning directory: {e}")
try:
subprocess.run(['iptables', '-t', 'nat', '-F', 'WGWADM_POSTROUTING'], capture_output=True)
subprocess.run(['iptables', '-t', 'nat', '-F', 'WGWADM_PREROUTING'], capture_output=True)
subprocess.run(['iptables', '-t', 'filter', '-F', 'WGWADM_FORWARD'], capture_output=True)
except Exception as e:
logger.error(f"Error flushing firewall: {e}")
def interface_is_up(interface):
return run_command(['wg', 'show', interface], check=False).returncode == 0
def get_local_dns_version(self):
try:
version_file = os.path.join(DNS_DIR, 'config_version.conf')
if os.path.exists(version_file):
with open(version_file, 'r') as f:
for line in f:
if line.startswith('DNS_VERSION='):
return int(line.strip().split('=')[1])
except Exception as e:
logger.error(f"Error reading DNS version: {e}")
return 0
def get_status(self):
params = {
'token': TOKEN,
'worker_config_version': self.config_version,
'worker_dns_version': self.dns_version,
'worker_version': WORKER_VERSION
def interface_section(config):
"""Return the [Interface] part of a WireGuard config, used to decide between syncconf and restart."""
return config.split('[Peer]', 1)[0].strip()
def read_file(path):
try:
with open(path, 'r', encoding='utf-8') as config_file:
return config_file.read()
except FileNotFoundError:
return None
def write_file(path, content, mode=0o600, in_place=False):
if in_place:
with open(path, 'w', encoding='utf-8') as config_file:
config_file.write(content)
os.chmod(path, mode)
return
temp_path = f'{path}.tmp'
with open(temp_path, 'w', encoding='utf-8') as config_file:
config_file.write(content)
os.chmod(temp_path, mode)
os.replace(temp_path, path)
def wireguard_status():
result = run_command(['wg', 'show', 'all', 'dump'], check=False)
if result.returncode != 0:
return {}
data = {}
# Peer lines: interface, public_key, preshared_key, endpoint, allowed_ips, latest_handshake, rx, tx, keepalive
for line in result.stdout.strip().splitlines():
parts = line.split('\t')
if len(parts) < 9:
continue
interface, peer_public_key, endpoint, allowed_ips = parts[0], parts[1], parts[3], parts[4]
data.setdefault(interface, {})[peer_public_key] = {
'allowed-ips': [ip for ip in allowed_ips.split(',') if ip] if allowed_ips else [],
'latest-handshakes': parts[5] or '',
'transfer': {'tx': int(parts[7] or 0), 'rx': int(parts[6] or 0)},
'endpoints': endpoint or '',
}
logger.info(f"Requesting status from Master... (Config Version: {self.config_version}, DNS Version: {self.dns_version}, Worker Version: {WORKER_VERSION})")
try:
response = self.session.get(f"{self.base_url}/status/", params=params, timeout=REQUEST_TIMEOUT)
logger.info(f"Status response received. HTTP Code: {response.status_code}")
return response
except requests.RequestException as e:
logger.error(f"Connection error while getting status: {e}")
return None
return data
def download_configs(self):
params = {
'token': TOKEN,
'worker_config_version': self.config_version,
'worker_dns_version': self.dns_version,
'worker_version': WORKER_VERSION
def gateway_enabled():
return os.path.isdir(GATEWAY_DIR) and os.path.isdir(CERTIFICATES_DIR)
def http_service_is_running(url):
try:
requests.get(url, timeout=3)
return True
except requests.RequestException:
return False
def _parse_not_after(value):
return datetime.strptime(value, '%b %d %H:%M:%S %Y %Z').replace(tzinfo=timezone.utc).isoformat()
def _fetch_certificate(hostname, verify):
if verify:
context = ssl.create_default_context()
else:
context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
with socket.create_connection(('127.0.0.1', 443), timeout=3) as raw_socket:
with context.wrap_socket(raw_socket, server_hostname=hostname) as tls_socket:
return tls_socket.getpeercert() if verify else tls_socket.getpeercert(binary_form=True)
def inspect_certificate(hostname):
"""Same checks as app_gateway/certificate_status.py on the primary server, against the local Caddy."""
try:
certificate = _fetch_certificate(hostname, verify=True)
issuer = dict(field[0] for field in certificate.get('issuer', ()))
return {
'status': 'trusted',
'issuer': issuer.get('organizationName') or issuer.get('commonName') or '',
'not_after': _parse_not_after(certificate['notAfter']),
}
try:
response = self.session.get(f"{self.base_url}/worker/get_config_files/", params=params, timeout=REQUEST_TIMEOUT)
if response.status_code == 200:
data = response.json()
if data.get('status') == 'success':
return data
return None
except requests.RequestException as e:
logger.error(f"Error downloading configs: {e}")
return None
except ssl.SSLCertVerificationError:
pass
except (OSError, ValueError):
return {'status': 'unavailable', 'issuer': '', 'not_after': None}
def download_dns_config(self):
params = {
'token': TOKEN,
'worker_config_version': self.config_version,
'worker_dns_version': self.dns_version,
'worker_version': WORKER_VERSION
}
try:
logger.info(f"Downloading DNS config... (Current Version: {self.dns_version})")
response = self.session.get(f"{self.base_url}/worker/get_dnsmasq_config/", params=params, timeout=REQUEST_TIMEOUT)
if response.status_code == 200:
tar_path = 'dnsmasq_config.tar.gz'
with open(tar_path, 'wb') as f:
f.write(response.content)
logger.info("Extracting DNS config...")
if not os.path.exists(DNS_DIR):
os.makedirs(DNS_DIR, exist_ok=True)
subprocess.run(['tar', 'xvfz', tar_path, '-C', DNS_DIR], check=True, capture_output=True)
if os.path.exists(tar_path):
os.remove(tar_path)
# Update version
self.dns_version = self.get_local_dns_version()
logger.info(f"DNS config updated (New Version: {self.dns_version})")
return True
else:
logger.error(f"Failed to download DNS config. Status: {response.status_code}")
return False
except Exception as e:
logger.error(f"Error updating DNS config: {e}")
return False
try:
certificate_der = _fetch_certificate(hostname, verify=False)
result = subprocess.run(
['openssl', 'x509', '-inform', 'DER', '-noout', '-issuer', '-enddate', '-nameopt', 'RFC2253'],
input=certificate_der, capture_output=True, timeout=5,
)
except (OSError, ValueError, subprocess.SubprocessError):
return {'status': 'unavailable', 'issuer': '', 'not_after': None}
issuer = ''
not_after = None
for line in result.stdout.decode(errors='replace').splitlines():
if line.startswith('issuer='):
parts = dict(part.split('=', 1) for part in line[len('issuer='):].split(',') if '=' in part)
issuer = INTERNAL_CA_NAME if INTERNAL_CA_NAME in line else parts.get('O') or parts.get('CN') or ''
elif line.startswith('notAfter='):
not_after = _parse_not_after(line[len('notAfter='):])
return {'status': 'untrusted', 'issuer': issuer, 'not_after': not_after}
def send_ping(self):
params = {
'token': TOKEN,
'worker_config_version': self.config_version,
'worker_dns_version': self.dns_version,
'worker_version': WORKER_VERSION
}
try:
logger.info("Sending ping to Master...")
self.session.get(f"{self.base_url}/worker/ping/", params=params, timeout=REQUEST_TIMEOUT)
except Exception as e:
logger.error(f"Error sending ping: {e}")
def apply_configs(self, data):
logger.info("Applying new configurations...")
files = data.get('files', {})
cluster_settings = data.get('cluster_settings', {})
new_config_version = cluster_settings.get('config_version', 0)
# 1. Stop existing interfaces
self.cleanup_wireguard()
# 2. Write new files
for filename, content in files.items():
if filename == 'wg-firewall.sh' and isinstance(content, str):
content = content.replace('wireguard-webadmin-dns', 'cluster-node-dns')
filepath = os.path.join(WIREGUARD_DIR, filename)
with open(filepath, 'w') as f:
f.write(content)
if filename == 'wg-firewall.sh':
os.chmod(filepath, 0o755)
# Start interfaces
conf_files = glob.glob(f"{WIREGUARD_DIR}/*.conf")
for conf in conf_files:
interface = os.path.basename(conf).replace('.conf', '')
logger.info(f"Starting WireGuard interface: {interface}")
def dns_is_running():
# dnsmasq always listens on loopback when it is restricted to specific interfaces. The DNS container pauses
# for a few seconds while reloading, so a missing answer is retried before reporting it as stopped.
query = b'\x12\x34\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x09localhost\x00\x00\x01\x00\x01'
for attempt in range(4):
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as dns_socket:
dns_socket.settimeout(2)
try:
subprocess.run(['wg-quick', 'up', interface], check=True, capture_output=True)
except subprocess.CalledProcessError as e:
logger.error(f"Failed to start {interface}: {e.stderr.decode()}")
dns_socket.sendto(query, ('127.0.0.1', 53))
dns_socket.recvfrom(512)
return True
except OSError:
pass
time.sleep(1)
return False
# 4. Update config version
self.config_version = new_config_version
logger.info(f"Configuration updated to version {self.config_version}")
class ClusterNodeAgent:
def __init__(self):
self.base_url = f"{MASTER_SERVER_ADDRESS.rstrip('/')}/api/cluster/v2"
self.session = requests.Session()
self.session.headers.update({
'Authorization': f'Bearer {TOKEN}',
'X-Worker-Version': str(WORKER_VERSION),
})
# Nothing is persisted: after a restart interfaces and firewall rules are gone, so every section is applied again
self.applied_hashes = {}
self.stats_interval = DEFAULT_STATS_INTERVAL
self.wireguard_active = False
def request(self, method, path, **kwargs):
try:
return self.session.request(method, f'{self.base_url}/{path}', timeout=REQUEST_TIMEOUT, **kwargs)
except requests.RequestException as error:
logger.error('Request to %s failed: %s', path, error)
return None
# Section handlers
def apply_firewall(self, section):
content = section['files'].get('wg-firewall.sh')
if content is None:
raise SectionError('wg-firewall.sh missing from section')
write_file(FIREWALL_SCRIPT, content, mode=0o700)
run_command(['bash', FIREWALL_SCRIPT])
def apply_wireguard(self, section):
desired_files = {
filename: content for filename, content in section['files'].items()
if filename.startswith('wg') and filename.endswith('.conf') and '/' not in filename
}
errors = []
for config_path in glob.glob(os.path.join(WIREGUARD_DIR, 'wg[0-9]*.conf')):
filename = os.path.basename(config_path)
if filename in desired_files:
continue
interface = filename[:-len('.conf')]
if interface_is_up(interface):
run_command(['wg-quick', 'down', interface], check=False)
os.remove(config_path)
logger.info('Removed WireGuard interface %s', interface)
for filename, content in sorted(desired_files.items()):
interface = filename[:-len('.conf')]
config_path = os.path.join(WIREGUARD_DIR, filename)
previous_content = read_file(config_path)
write_file(config_path, content)
try:
if not interface_is_up(interface):
run_command(['wg-quick', 'up', interface])
logger.info('Started WireGuard interface %s', interface)
elif previous_content is None or interface_section(previous_content) != interface_section(content):
run_command(['wg-quick', 'down', interface], check=False)
run_command(['wg-quick', 'up', interface])
logger.info('Restarted WireGuard interface %s', interface)
elif previous_content != content:
self.sync_peers(interface, config_path)
logger.info('Synchronized peers of WireGuard interface %s', interface)
except SectionError as error:
errors.append(str(error))
self.wireguard_active = True
if errors:
raise SectionError('; '.join(errors))
@staticmethod
def sync_peers(interface, config_path):
stripped_config = run_command(['wg-quick', 'strip', config_path]).stdout
with tempfile.NamedTemporaryFile('w', suffix='.conf', delete=False) as stripped_file:
stripped_file.write(stripped_config)
try:
run_command(['wg', 'syncconf', interface, stripped_file.name])
finally:
os.remove(stripped_file.name)
def apply_dns(self, section):
files = section['files']
if DNS_MAIN_CONFIG not in files:
raise SectionError(f'{DNS_MAIN_CONFIG} missing from section')
os.makedirs(DNS_DIR, exist_ok=True)
for filename, content in files.items():
if filename != DNS_MAIN_CONFIG and '/' not in filename:
write_file(os.path.join(DNS_DIR, filename), content, mode=0o644)
for config_path in glob.glob(os.path.join(DNS_DIR, '*.conf')):
if os.path.basename(config_path) not in files:
os.remove(config_path)
# Written last and in place: the DNS container watches this file with inotify to reload
write_file(os.path.join(DNS_DIR, DNS_MAIN_CONFIG), files[DNS_MAIN_CONFIG], mode=0o644, in_place=True)
def apply_settings(self, section):
self.stats_interval = int(section['values'].get('stats_sync_interval', DEFAULT_STATS_INTERVAL))
def apply_certificates(self, section):
if not gateway_enabled():
return
files = {
filename: content for filename, content in section['files'].items()
if CERTIFICATE_FILE_PATTERN.match(filename)
}
for filename, content in files.items():
certificate_path = os.path.join(CERTIFICATES_DIR, filename)
if read_file(certificate_path) != content:
write_file(certificate_path, content, mode=0o600)
for certificate_path in glob.glob(os.path.join(CERTIFICATES_DIR, '*')):
if os.path.basename(certificate_path) not in files:
os.remove(certificate_path)
def apply_gateway(self, section):
if not gateway_enabled():
return
for filename in GATEWAY_FILES:
if filename not in section['files']:
raise SectionError(f'{filename} missing from section')
for filename in GATEWAY_FILES:
write_file(os.path.join(GATEWAY_DIR, filename), section['files'][filename], mode=0o644)
SECTION_HANDLERS = {
'firewall': apply_firewall,
'wireguard': apply_wireguard,
'dns': apply_dns,
'settings': apply_settings,
'certificates': apply_certificates,
'gateway': apply_gateway,
}
# Main loop
def shutdown_wireguard(self):
if not self.wireguard_active:
return
logger.warning('Shutting down WireGuard interfaces and firewall rules')
for config_path in glob.glob(os.path.join(WIREGUARD_DIR, 'wg[0-9]*.conf')):
run_command(['wg-quick', 'down', os.path.basename(config_path)[:-len('.conf')]], check=False)
for table, chain in FIREWALL_CHAINS:
run_command(['iptables', '-t', table, '-F', chain], check=False)
self.applied_hashes.pop('wireguard', None)
self.applied_hashes.pop('firewall', None)
self.wireguard_active = False
def sync_configuration(self):
response = self.request('GET', 'manifest/')
if response is None:
return True
if response.status_code == 403:
logger.error('Token rejected by the primary server. Stopping.')
self.shutdown_wireguard()
return False
if response.status_code == 400:
logger.warning('Primary server refused this worker: %s', response.json().get('message', ''))
self.shutdown_wireguard()
return True
if response.status_code != 200:
logger.error('Unexpected manifest response: HTTP %s', response.status_code)
return True
manifest = response.json().get('sections', {})
reports = {}
for section_name, section_hash in manifest.items():
if section_name not in self.SECTION_HANDLERS or self.applied_hashes.get(section_name) == section_hash:
continue
section_response = self.request('GET', f'section/{section_name}/')
if section_response is None or section_response.status_code != 200:
continue
section = section_response.json()
try:
self.SECTION_HANDLERS[section_name](self, section)
self.applied_hashes[section_name] = section['hash']
reports[section_name] = {'hash': section['hash'], 'status': 'ok'}
logger.info('Section %s applied', section_name)
except (SectionError, OSError, ValueError) as error:
# Forget the last applied content: the next version must be applied and reported even if it
# is identical to the one that was applied before the failure
self.applied_hashes.pop(section_name, None)
reports[section_name] = {'hash': section['hash'], 'status': 'error', 'message': str(error)}
logger.error('Section %s failed: %s', section_name, error)
if reports:
# Services are reported with the periodic stats: right after a DNS change dnsmasq is still reloading
self.request('POST', 'report/', json={'sections': reports})
return True
@staticmethod
def services_status():
services = {'dnsmasq': 'running' if dns_is_running() else 'stopped'}
if gateway_enabled():
services['caddy'] = 'running' if http_service_is_running(CADDY_ADMIN_URL) else 'stopped'
services['auth_gateway'] = 'running' if http_service_is_running(AUTH_GATEWAY_URL) else 'stopped'
return services
@staticmethod
def certificates_status():
try:
with open(os.path.join(GATEWAY_DIR, 'tls.json'), 'r', encoding='utf-8') as tls_file:
hosts = json.load(tls_file).get('hosts', {})
except (OSError, ValueError):
return {}
return {hostname: inspect_certificate(hostname) for hostname in sorted(hosts)}
def send_stats(self):
try:
stats = self.func_process_wireguard_status()
params = {
'token': TOKEN,
'worker_config_version': self.config_version,
'worker_dns_version': self.dns_version,
'worker_version': WORKER_VERSION
}
logger.info("Sending WireGuard stats to Master...")
response = self.session.post(f"{self.base_url}/worker/submit_wireguard_stats/", json=stats, params=params, timeout=REQUEST_TIMEOUT)
if response.status_code == 200:
logger.info("Stats sent successfully.")
return True
else:
logger.error(f"Failed to send stats. Status: {response.status_code}")
return False
except Exception as e:
logger.error(f"Error sending stats: {e}")
return False
report = {
'wireguard_status': wireguard_status(),
'services': self.services_status(),
}
if gateway_enabled():
report['certificates'] = self.certificates_status()
self.request('POST', 'report/', json=report)
@staticmethod
def publish_network_namespace():
# Containers sharing this network namespace (DNS, Caddy) restart themselves when it changes
network_namespace = os.readlink('/proc/self/ns/net')
for directory in (DNS_DIR, GATEWAY_DIR):
if os.path.isdir(directory):
write_file(os.path.join(directory, '.netns_owner'), network_namespace + '\n', mode=0o644)
def run(self):
if not self.should_run:
return
logger.info("Cluster Worker starting...")
# Initial cleanup
self.cleanup_wireguard()
last_config_check = 0
last_stats_send = 0
stats_sync_interval = 60 # Default initial value
logger.info('Cluster node agent %s starting', WORKER_VERSION)
self.publish_network_namespace()
self.wireguard_active = True
self.shutdown_wireguard()
last_manifest = 0
last_stats = 0
while True:
current_time = time.time()
# Check Config (Fixed 60s interval)
if current_time - last_config_check >= 60:
try:
response = self.get_status()
last_config_check = time.time()
if response is not None:
if response.status_code == 403:
logger.error("Received 403 Forbidden (Token invalid/deleted). Deactivating WireGuard and stopping requests permanently.")
self.cleanup_wireguard()
self.config_version = 0
self.should_run = False
break
if response.status_code == 400:
logger.warning("Received 400 Bad Request (Worker suspended or error). Deactivating WireGuard and Firewall, but will keep retrying...")
self.cleanup_wireguard()
self.config_version = 0
if response.status_code == 200:
data = response.json()
# Update stats interval from master settings
cluster_settings = data.get('cluster_settings', {})
stats_sync_interval = cluster_settings.get('stats_sync_interval', 60)
remote_config_version = cluster_settings.get('config_version', 0)
# Check WireGuard Config
updated = False
if int(remote_config_version) != self.config_version:
logger.info(f"Config version mismatch (Local: {self.config_version}, Remote: {remote_config_version}). Updating...")
config_data = self.download_configs()
if config_data:
self.apply_configs(config_data)
self.send_ping()
updated = True
else:
logger.error("Failed to download config files.")
# Check DNS Config
remote_dns_version = int(cluster_settings.get('dns_version', 0))
if not updated and remote_dns_version != self.dns_version:
logger.info(f"DNS version mismatch (Local: {self.dns_version}, Remote: {remote_dns_version}). Updating...")
if self.download_dns_config():
self.send_ping()
updated = True
if not updated and int(remote_config_version) == self.config_version and remote_dns_version == self.dns_version:
logger.info(f"No changes detected. Configuration is up to date (WG: {self.config_version}, DNS: {self.dns_version}).")
except Exception as e:
logger.error(f"Unexpected error in config check loop: {e}")
# Check Stats Sync
if current_time - last_stats_send >= stats_sync_interval:
now = time.monotonic()
if now - last_manifest >= MANIFEST_INTERVAL or not last_manifest:
last_manifest = now
if not self.sync_configuration():
break
if now - last_stats >= self.stats_interval:
last_stats = now
self.send_stats()
last_stats_send = time.time()
# Sleep briefly to be responsive but not busy loop
time.sleep(1)
# Final loop state if 403 was received
while not self.should_run:
logger.info("Worker disabled due to 403 Forbidden. WireGuard is off.")
time.sleep(60)
while True:
time.sleep(3600)
if __name__ == "__main__":
worker = ClusterWorker()
worker.run()
if __name__ == '__main__':
if not MASTER_SERVER_ADDRESS or not TOKEN:
logger.error('MASTER_SERVER_ADDRESS and TOKEN must be set')
else:
ClusterNodeAgent().run()
+1 -1
View File
@@ -1,7 +1,7 @@
#!/bin/bash
set -e
if [[ "$COMPOSE_VERSION" != "c1b" ]]; then
if [[ "$COMPOSE_VERSION" != "c2a" ]]; then
echo "ERROR: Please upgrade your docker compose file. Exiting."
exit 1
fi
+17
View File
@@ -34,7 +34,24 @@ handle_sigint() {
exit 0
}
# Written by cluster nodes, where this container shares the network namespace of the node. When the node
# container is restarted it gets a new namespace, so this container must restart to join it again.
NETNS_OWNER_FILE="/etc/dnsmasq/.netns_owner"
watch_network_namespace() {
while sleep 30; do
if [ -f "$NETNS_OWNER_FILE" ] && [ "$(cat "$NETNS_OWNER_FILE")" != "$(readlink /proc/self/ns/net)" ]; then
echo "Network namespace of the cluster node changed, restarting..."
kill -TERM 1
pkill inotifywait
return
fi
done
}
trap handle_sigint SIGINT
trap 'pkill dnsmasq; exit 1' SIGTERM
create_default_config
watch_network_namespace &
start_dnsmasq
+12 -52
View File
@@ -1,58 +1,8 @@
import os
import tarfile
from cluster.models import ClusterSettings
from .models import DNSSettings, StaticHost, DNSFilterList
def compress_dnsmasq_config():
output_file = "/etc/dnsmasq/dnsmasq_config.tar.gz"
base_dir = "/etc/dnsmasq"
cluster_settings = ClusterSettings.objects.filter(enabled=True, name='cluster_settings').first()
if not cluster_settings:
if os.path.exists(output_file):
os.remove(output_file)
return None
if not os.path.isdir(base_dir):
if os.path.exists(output_file):
os.remove(output_file)
return None
conf_files = [
fn for fn in os.listdir(base_dir)
if fn.endswith(".conf") and os.path.isfile(os.path.join(base_dir, fn))
]
# If tar exists and is newer (or equal) than all .conf, do not recompile
if os.path.exists(output_file):
tar_mtime = os.path.getmtime(output_file)
newest_conf_mtime = max(
os.path.getmtime(os.path.join(base_dir, fn)) for fn in conf_files
)
if newest_conf_mtime <= tar_mtime:
return output_file
# If we reach here, we need to increment DNS version and recompile the tar.gz
cluster_settings.dns_version += 1
cluster_settings.save()
dns_version_file = os.path.join(base_dir, "config_version.conf")
with open(dns_version_file, "w", encoding="utf-8") as f:
f.write(f"DNS_VERSION={cluster_settings.dns_version}\n")
if "config_version.conf" not in conf_files:
conf_files.append("config_version.conf")
# Create tar.gz
tmp_output = output_file + ".tmp"
with tarfile.open(tmp_output, "w:gz") as tar:
for fn in conf_files:
fullpath = os.path.join(base_dir, fn)
tar.add(fullpath, arcname=fn)
os.replace(tmp_output, output_file)
return output_file
def generate_unbound_config():
dns_settings = DNSSettings.objects.get(name='dns_settings')
static_hosts = StaticHost.objects.all()
@@ -106,11 +56,21 @@ def generate_dnsdist_config():
return dnsdist_config
def generate_dnsmasq_config():
def generate_dnsmasq_config(cluster_node=False):
dns_settings = DNSSettings.objects.get(name='dns_settings')
static_hosts = StaticHost.objects.all()
dns_lists = DNSFilterList.objects.filter(enabled=True)
dnsmasq_config = f'''
if cluster_node:
# On cluster nodes dnsmasq shares the network namespace of the node and must also answer on
# WireGuard interfaces created after it starts
dnsmasq_config = '''
no-dhcp-interface=
interface=wg*
bind-dynamic
'''
else:
dnsmasq_config = '''
no-dhcp-interface=
listen-address=0.0.0.0
bind-interfaces
+1 -2
View File
@@ -13,7 +13,7 @@ from django.utils.translation import gettext_lazy as _
from user_manager.models import UserAcl
from .forms import DNSFilterListForm
from .forms import DNSSettingsForm, StaticHostForm
from .functions import generate_dnsmasq_config, compress_dnsmasq_config
from .functions import generate_dnsmasq_config
from .models import DNSFilterList, DNSSettings
from .models import StaticHost
@@ -44,7 +44,6 @@ def export_dns_configuration():
dnsmasq_config = generate_dnsmasq_config()
with open(settings.DNS_CONFIG_FILE, 'w') as f:
f.write(dnsmasq_config)
compress_dnsmasq_config()
dns_settings, dns_settings_created = DNSSettings.objects.get_or_create(name='dns_settings')
dns_settings.pending_changes = False
dns_settings.save(update_fields=['pending_changes', 'updated'])
+3
View File
@@ -24,6 +24,7 @@ services:
- app_secrets:/app_secrets/
- rrd_data:/rrd_data/
- caddy_json_export:/caddy_json_export/
- caddy_cert_export:/caddy_cert_export/:ro
ports:
# Do not directly expose the Django port to the internet, use the reverse proxy below instead
# - "127.0.0.1:8000:8000"
@@ -87,6 +88,7 @@ services:
volumes:
- static_volume:/static
- caddy_json_export:/caddy_json_export
- caddy_cert_export:/caddy_cert_export
- caddy_data:/data
- caddy_config:/config
network_mode: "service:wireguard-webadmin"
@@ -115,6 +117,7 @@ volumes:
app_secrets:
rrd_data:
caddy_json_export:
caddy_cert_export:
auth_gateway_data:
caddy_data:
caddy_config:
+3
View File
@@ -27,6 +27,7 @@ services:
- app_secrets:/app_secrets/
- rrd_data:/rrd_data/
- caddy_json_export:/caddy_json_export/
- caddy_cert_export:/caddy_cert_export/:ro
ports:
# Do not directly expose the Django port to the internet, use some kind of reverse proxy with SSL.
- "8000:8000"
@@ -99,6 +100,7 @@ services:
volumes:
- static_volume:/static
- caddy_json_export:/caddy_json_export
- caddy_cert_export:/caddy_cert_export
- caddy_data:/data
- caddy_config:/config
network_mode: "service:wireguard-webadmin"
@@ -130,6 +132,7 @@ volumes:
app_secrets:
rrd_data:
caddy_json_export:
caddy_cert_export:
auth_gateway_data:
caddy_data:
caddy_config:
+32 -11
View File
@@ -6,6 +6,9 @@ from django.utils.translation import gettext_lazy as _
from firewall.models import FirewallRule, FirewallSettings, RedirectRule
from wireguard.models import PeerAllowedIP, WireGuardInstance, Peer
PRIMARY_HOSTNAME = 'wireguard-webadmin'
CLUSTER_NODE_HOSTNAME = 'cluster-node'
def get_peer_addresses(peers, include_networks):
addresses = []
@@ -213,19 +216,21 @@ def generate_route_policy_rules():
return route_policy_rules
def generate_firewall_header():
firewall_settings, firewall_settings_created = FirewallSettings.objects.get_or_create(name='global')
header = f'''#!/bin/bash
def generate_firewall_header(cluster_node=False):
header = '''#!/bin/bash
# Description: Firewall rules for WireGuard_WebAdmin
# Do not edit this file directly. Use the web interface to manage firewall rules.
#
# This script was generated by WireGuard_WebAdmin on {timezone.now().strftime('%Y-%m-%d %H:%M:%S %Z')}
#
DNS_IP=$(getent hosts wireguard-webadmin-dns | awk '{{ print $1 }}')
'''
if not cluster_node:
# Cluster nodes compare scripts by hash, so their copy must not change on every generation
header += f"# This script was generated by WireGuard_WebAdmin on {timezone.now().strftime('%Y-%m-%d %H:%M:%S %Z')}\n#\n"
header += '''DNS_IP=$(getent hosts wireguard-webadmin-dns | awk '{ print $1 }')
if [ -z "$DNS_IP" ]; then
DNS_IP="127.0.0.250"
fi
'''
header += '''
iptables -t nat -N WGWADM_POSTROUTING >> /dev/null 2>&1
iptables -t nat -N WGWADM_PREROUTING >> /dev/null 2>&1
iptables -t filter -N WGWADM_FORWARD >> /dev/null 2>&1
@@ -249,7 +254,7 @@ iptables -t filter -A FORWARD -m state --state RELATED,ESTABLISHED -j ACCEPT
iptables -t filter -A FORWARD -i wg+ -j WGWADM_ROUTE_POLICY
iptables -t filter -A FORWARD -j WGWADM_FORWARD
'''
if not settings.VPN_CLIENTS_CAN_ACCESS_DJANGO:
if not cluster_node and not settings.VPN_CLIENTS_CAN_ACCESS_DJANGO:
header += 'iptables -t filter -A INPUT -i wg+ -p tcp --dport 8000 -j REJECT\n'
return header
@@ -276,7 +281,7 @@ def generate_firewall_footer():
return footer
def generate_port_forward_firewall():
def generate_port_forward_firewall(server_hostname=PRIMARY_HOSTNAME):
firewall_settings, firewall_settings_created = FirewallSettings.objects.get_or_create(name='global')
redirect_firewall = ''
wan_interface = firewall_settings.wan_interface
@@ -298,7 +303,7 @@ def generate_port_forward_firewall():
rule_destination = peer_allowed_ip_address.allowed_ip
if rule_destination:
rule_text = f"# {redirect_rule.port}/{redirect_rule.protocol} - {redirect_rule.uuid} - Port Forward Rule set{description}\n"
rule_text += f"iptables -t nat -A WGWADM_PREROUTING -p {redirect_rule.protocol} -d wireguard-webadmin -i {wan_interface} --dport {redirect_rule.port} -j DNAT --to-dest {rule_destination}:{destination_port}\n"
rule_text += f"iptables -t nat -A WGWADM_PREROUTING -p {redirect_rule.protocol} -d {server_hostname} -i {wan_interface} --dport {redirect_rule.port} -j DNAT --to-dest {rule_destination}:{destination_port}\n"
if redirect_rule.masquerade_source:
rule_text += f"iptables -t nat -A WGWADM_POSTROUTING -p {redirect_rule.protocol} -d {rule_destination} -o wg+ --dport {destination_port} -j MASQUERADE\n"
@@ -312,4 +317,20 @@ def generate_port_forward_firewall():
rule_text = f"# {redirect_rule.port}/{redirect_rule.protocol} - {redirect_rule.uuid} - Port Forward Rule set{description} - Missing IP for selected peer: {redirect_rule.peer}\n"
redirect_firewall += rule_text
return redirect_firewall
return redirect_firewall
def generate_firewall_script(cluster_node=False):
"""
Build wg-firewall.sh for the primary server or for a cluster node. On cluster nodes dnsmasq shares the
network namespace of the node, so DNS is answered directly on the instance address without DNAT.
"""
firewall_content = generate_firewall_header(cluster_node=cluster_node)
if not cluster_node:
firewall_content += generate_redirect_dns_rules()
firewall_content += generate_port_forward_firewall(
server_hostname=CLUSTER_NODE_HOSTNAME if cluster_node else PRIMARY_HOSTNAME
)
firewall_content += generate_route_policy_rules()
firewall_content += export_user_firewall()
firewall_content += generate_firewall_footer()
return firewall_content
+13 -1
View File
@@ -10,7 +10,7 @@ from django.shortcuts import render, get_object_or_404, redirect
from django.urls import reverse
from django.utils.translation import gettext as _
from app_gateway.caddy_config_export import caddy_export_pending
from app_gateway.caddy_config_export import auto_apply_caddy_config, caddy_export_pending
from app_gateway.setup_defaults import create_default_entries
from gatekeeper.forms import GatekeeperUserForm, GatekeeperGroupForm, AuthMethodForm, AuthMethodAllowedDomainForm, \
AuthMethodAllowedEmailForm, GatekeeperIPAddressForm
@@ -68,6 +68,7 @@ def view_manage_gatekeeper_user(request):
if form.is_valid():
form.save()
messages.success(request, _('Gatekeeper User saved successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
form_description = {
@@ -110,6 +111,7 @@ def view_delete_gatekeeper_user(request):
if request.method == 'POST':
gatekeeper_user.delete()
messages.success(request, _('Gatekeeper User deleted successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
context = {
@@ -141,6 +143,7 @@ def view_manage_gatekeeper_group(request):
if form.is_valid():
form.save()
messages.success(request, _('Gatekeeper Group saved successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
context = {
@@ -163,6 +166,7 @@ def view_delete_gatekeeper_group(request):
if request.method == 'POST':
gatekeeper_group.delete()
messages.success(request, _('Gatekeeper Group deleted successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
context = {
@@ -194,6 +198,7 @@ def view_manage_auth_method(request):
if form.is_valid():
form.save()
messages.success(request, _('Authentication Method saved successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
form_description = {
@@ -243,6 +248,7 @@ def view_delete_auth_method(request):
if request.method == 'POST':
auth_method.delete()
messages.success(request, _('Authentication Method deleted successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
context = {
@@ -313,6 +319,7 @@ def view_manage_auth_domain(request):
if form.is_valid():
form.save()
messages.success(request, _('Allowed Domain saved successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
context = {
@@ -335,6 +342,7 @@ def view_delete_auth_domain(request):
if request.method == 'POST':
auth_domain.delete()
messages.success(request, _('Allowed Domain deleted successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
context = {
@@ -366,6 +374,7 @@ def view_manage_auth_email(request):
if form.is_valid():
form.save()
messages.success(request, _('Allowed Email saved successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
context = {
@@ -388,6 +397,7 @@ def view_delete_auth_email(request):
if request.method == 'POST':
auth_email.delete()
messages.success(request, _('Allowed Email deleted successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
context = {
@@ -419,6 +429,7 @@ def view_manage_gatekeeper_ip(request):
if form.is_valid():
form.save()
messages.success(request, _('IP Address saved successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
form_description = {
@@ -460,6 +471,7 @@ def view_delete_gatekeeper_ip(request):
if request.method == 'POST':
gatekeeper_ip.delete()
messages.success(request, _('IP Address deleted successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
context = {
+5
View File
@@ -1,6 +1,11 @@
#!/bin/bash
set -e
# Containers sharing this network namespace (Caddy) restart themselves when it changes
if [ -d /caddy_json_export ]; then
readlink /proc/self/ns/net > /caddy_json_export/.netns_owner
fi
# Lets wait for the DNS container to start
sleep 5
+11 -10
View File
@@ -195,17 +195,18 @@
$.getJSON("{% url 'application_certificate_status' %}?uuid={{ application.uuid }}", function (data) {
$('.certificate-status').each(function () {
var cell = $(this);
var hostStatus = data.hosts[cell.data('host-uuid')];
var serversStatus = data.hosts[cell.data('host-uuid')] || [];
cell.empty();
if (!hostStatus) {
return;
}
cell.append(
$('<span class="badge"></span>')
.addClass('badge-' + hostStatus.css_class)
.attr('title', hostStatus.detail)
.text(hostStatus.label)
);
$.each(serversStatus, function (index, serverStatus) {
var title = serverStatus.server ? serverStatus.server + ': ' + serverStatus.detail : serverStatus.detail;
var label = serverStatus.server ? serverStatus.server + ': ' + serverStatus.label : serverStatus.label;
cell.append(
$('<span class="badge mr-1"></span>')
.addClass('badge-' + serverStatus.css_class)
.attr('title', title)
.text(label)
);
});
});
}).fail(function () {
$('.certificate-status').html('<span class="text-muted">-</span>');
+168
View File
@@ -0,0 +1,168 @@
{% extends "base.html" %}
{% load i18n %}
{% block content %}
<div class="card card-primary card-outline">
<div class="card-header">
<h3 class="card-title">
<i class="fas fa-server"></i> {{ worker.name }}
{% if not worker.enabled %}
<span class="badge badge-secondary">{% trans 'Disabled' %}</span>
{% elif worker.error_status %}
<span class="badge badge-danger">{% trans 'Refused' %}</span>
{% elif worker.is_online %}
<span class="badge badge-success">{% trans 'Online' %}</span>
{% else %}
<span class="badge badge-danger">{% trans 'Offline' %}</span>
{% endif %}
</h3>
<div class="card-tools">
<a href="{% url 'cluster_main' %}" class="btn btn-sm btn-secondary"><i class="fas fa-arrow-left"></i> {% trans 'Back to List' %}</a>
<a href="{% url 'worker_manage' %}?uuid={{ worker.uuid }}" class="btn btn-sm btn-info"><i class="fas fa-edit"></i> {% trans 'Edit Worker' %}</a>
</div>
</div>
<div class="card-body">
{% if worker.error_status %}
<div class="alert alert-danger"><i class="icon fas fa-ban"></i> {{ worker.get_error_status_display }}</div>
{% endif %}
{% if worker.enabled and not worker.error_status and not worker.is_online %}
<div class="alert alert-warning"><i class="icon fas fa-plug"></i> {% trans 'The worker is offline. The information below is the last state it reported.' %}</div>
{% endif %}
{% if worker.outdated %}
<div class="alert alert-warning"><i class="icon fas fa-cloud-download-alt"></i> {% trans 'The worker is outdated. Please update it to the latest version.' %}</div>
{% endif %}
<div class="row">
<div class="col-md-6">
<dl class="row mb-0">
<dt class="col-sm-5">{% trans 'Hostname' %}</dt>
<dd class="col-sm-7">{{ worker.hostname|default:'-' }}</dd>
<dt class="col-sm-5">{% trans 'IP Address' %}</dt>
<dd class="col-sm-7">{{ worker.ip_address|default:'-' }} {% if worker.ip_lock %}<i class="fas fa-lock text-warning" title="{% trans 'IP Lock Enabled' %}"></i>{% endif %}</dd>
<dt class="col-sm-5">{% trans 'Location' %}</dt>
<dd class="col-sm-7">{{ worker.city|default:'' }}{% if worker.city and worker.country %}, {% endif %}{{ worker.country|default:'' }}{% if not worker.city and not worker.country %}-{% endif %}</dd>
</dl>
</div>
<div class="col-md-6">
<dl class="row mb-0">
<dt class="col-sm-5">{% trans 'Last Seen' %}</dt>
<dd class="col-sm-7">{% if worker.status %}{{ worker.status.last_seen|date:"SHORT_DATETIME_FORMAT" }}{% else %}{% trans 'Never' %}{% endif %}</dd>
<dt class="col-sm-5">{% trans 'Version' %}</dt>
<dd class="col-sm-7">{% if worker.status %}v{{ worker.status.worker_version }}{% else %}-{% endif %} <small class="text-muted">({% trans 'current' %}: v{{ current_worker_version }})</small></dd>
<dt class="col-sm-5">{% trans 'App Gateway' %}</dt>
<dd class="col-sm-7">{% if worker.runs_gateway %}<span class="badge badge-success">{% trans 'Enabled' %}</span>{% else %}<span class="badge badge-secondary">{% trans 'Not installed' %}</span>{% endif %}</dd>
</dl>
</div>
</div>
</div>
</div>
<div class="row">
<div class="col-lg-7">
<div class="card card-outline card-info">
<div class="card-header"><h3 class="card-title"><i class="fas fa-sync"></i> {% trans 'Synchronization' %}</h3></div>
<div class="card-body p-0">
<table class="table table-sm mb-0">
<thead><tr><th>{% trans 'Section' %}</th><th>{% trans 'Status' %}</th><th>{% trans 'Last update' %}</th></tr></thead>
<tbody>
{% for section in sections %}
<tr>
<td><code>{{ section.name }}</code></td>
<td>
{% if section.state == 'synced' %}<span class="badge badge-success">{% trans 'Synchronized' %}</span>
{% elif section.state == 'error' %}<span class="badge badge-danger">{% trans 'Error' %}</span>
{% else %}<span class="badge badge-secondary">{% trans 'Pending' %}</span>{% endif %}
{% if section.message %}<br><small class="text-muted text-break">{{ section.message|truncatechars:300 }}</small>{% endif %}
</td>
<td class="text-nowrap"><small>{{ section.updated|date:"SHORT_DATETIME_FORMAT"|default:'-' }}</small></td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
<div class="col-lg-5">
<div class="card card-outline card-info">
<div class="card-header"><h3 class="card-title"><i class="fas fa-heartbeat"></i> {% trans 'Services' %}</h3></div>
<div class="card-body">
{% for service, state in worker.services %}
<span class="badge {% if state == 'running' %}badge-success{% else %}badge-danger{% endif %} mr-1">{{ service }}: {{ state }}</span>
{% empty %}
<span class="text-muted">{% trans 'No report received yet.' %}</span>
{% endfor %}
</div>
</div>
{% if certificates %}
<div class="card card-outline card-info">
<div class="card-header"><h3 class="card-title"><i class="fas fa-certificate"></i> {% trans 'Certificates served' %}</h3></div>
<div class="card-body p-0">
<table class="table table-sm mb-0">
<tbody>
{% for certificate in certificates %}
<tr>
<td>{{ certificate.hostname }}</td>
<td>
{% if certificate.status == 'trusted' %}<span class="badge badge-success">{% trans 'Trusted' %}</span>
{% elif certificate.status == 'untrusted' %}<span class="badge badge-warning">{% trans 'Self-signed' %}</span>
{% else %}<span class="badge badge-danger">{% trans 'Unavailable' %}</span>{% endif %}
<small class="text-muted">{{ certificate.issuer }}{% if certificate.not_after %} · {{ certificate.not_after|date:"SHORT_DATE_FORMAT" }}{% endif %}</small>
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
{% endif %}
</div>
</div>
<div class="card card-outline card-secondary">
<div class="card-header"><h3 class="card-title"><i class="fas fa-download"></i> {% trans 'Installation' %}</h3></div>
<div class="card-body">
<p>{% blocktrans trimmed %}
Copy <code>cluster_node.yml</code> (and <code>cluster_node-gateway.yml</code> to publish App Gateway applications)
from the <a href="https://github.com/eduardogsilva/wireguard_webadmin" target="_blank" rel="noopener noreferrer">project repository</a>
to the worker, and create a <code>.env</code> file next to them:
{% endblocktrans %}</p>
<div class="position-relative">
<button type="button" class="btn btn-xs btn-outline-secondary position-absolute copy-button" style="top: 6px; right: 6px;" data-target="#worker-env" title="{% trans 'Copy' %}"><i class="far fa-copy"></i></button>
<pre id="worker-env" class="p-2 pr-5">MASTER_SERVER_ADDRESS={{ primary_address }}
TOKEN={{ worker.token }}
TIMEZONE={{ time_zone }}</pre>
</div>
<p class="mb-1"><strong>{% trans 'VPN only' %}</strong></p>
<div class="position-relative">
<button type="button" class="btn btn-xs btn-outline-secondary position-absolute copy-button" style="top: 6px; right: 6px;" data-target="#worker-vpn" title="{% trans 'Copy' %}"><i class="far fa-copy"></i></button>
<pre id="worker-vpn" class="p-2 pr-5">docker compose -f cluster_node.yml up -d</pre>
</div>
<p class="mb-1"><strong>{% trans 'VPN and App Gateway' %}</strong></p>
<div class="position-relative">
<button type="button" class="btn btn-xs btn-outline-secondary position-absolute copy-button" style="top: 6px; right: 6px;" data-target="#worker-gateway" title="{% trans 'Copy' %}"><i class="far fa-copy"></i></button>
<pre id="worker-gateway" class="p-2 pr-5" style="white-space: pre-wrap;">docker compose -f cluster_node.yml -f cluster_node-gateway.yml up -d</pre>
</div>
<ul class="text-muted mb-0">
<li>{% trans 'To update, run docker compose pull followed by the same up -d command, always with every compose file of the worker.' %}</li>
<li>{% trans 'Applications published by workers must use DNS validation or self-signed certificates: certificates are issued by this primary server and distributed to the workers.' %}</li>
<li>{% trans 'Publish the WireGuard ports (51820-51839/udp) and, with the App Gateway, ports 80 and 443 of the worker.' %}</li>
</ul>
</div>
</div>
{% endblock %}
{% block custom_page_scripts %}
<script>
$(document).ready(function () {
$('.copy-button').click(function () {
var button = $(this);
navigator.clipboard.writeText($(button.data('target')).text()).then(function () {
button.find('i').removeClass('fa-copy far').addClass('fas fa-check');
setTimeout(function () { button.find('i').removeClass('fas fa-check').addClass('far fa-copy'); }, 1500);
});
});
});
</script>
{% endblock %}
+190 -203
View File
@@ -2,222 +2,209 @@
{% load i18n %}
{% block content %}
<table class="table table-striped table-bordered">
<thead>
<tr>
<th>{% trans 'Name' %}</th>
<th>{% trans 'Status' %}</th>
<th>{% trans 'IP Address' %}</th>
<th><i class="fas fa-map-marker-alt" title="{% trans 'Location' %}"></i></th>
<th><i class="far fa-clock" title="{% trans 'Last Seen' %}"></i></th>
<th><i class="fas fa-cogs" title="{% trans 'Config Version' %}"></i></th>
<th><i class="far fa-edit"></i></th>
</tr>
</thead>
<tbody>
{% for worker in workers %}
<tr>
<td>{{ worker }}</td>
<td style="width: 1%; white-space: nowrap;">
{% if worker.error_status %}
<i class="fas fa-exclamation-triangle text-danger blink" title="{{ worker.get_error_status_display }}"></i>
{% else %}
{% if worker.enabled %}
{% if cluster_settings.config_version != worker.workerstatus.config_version or cluster_settings.dns_version != worker.workerstatus.dns_version %}
<i class="fas fa-sync text-primary blink" {% trans 'Configuration sync in progress' %}></i>
{% else %}
<i class="fas fa-check text-green"></i>
{% endif %}
{% else %}
<i class="fas fa-times text-gray"></i>
{% endif %}
{% endif %}
{% if current_worker_version != worker.workerstatus.worker_version %}
<i class="fas fa-cloud-download-alt text-primary blink" title="{% trans 'The worker is outdated. Please update it to the latest version.' %}"></i>
{% endif %}
</td>
<td>
{% if worker.ip_address %}
{{ worker.ip_address }}
{% if worker.ip_lock %}
<i class="fas fa-lock text-warning" title="{% trans 'IP Lock Enabled' %}"></i>
{% endif %}
{% else %}
<span class="text-muted">{% trans 'Not set' %}</span>
{% endif %}
</td>
<td>
{% if worker.country or worker.city %}
{% if worker.city %}{{ worker.city }}{% endif %}{% if worker.city and worker.country %}, {% endif %}{% if worker.country %}{{ worker.country }}{% endif %}
{% else %}
<span class="text-muted">{% trans 'Not set' %}</span>
{% endif %}
</td>
<td style="width: 1%; white-space: nowrap;" class="{% if not worker.is_online %}text-danger blink{% endif %}">
{% if worker.workerstatus %}
{{ worker.workerstatus.last_seen|date:"M d, H:i" }}
{% else %}
<span class="{% if not worker.is_online %}text-danger blink{% endif %}">{% trans 'Never' %}</span>
{% endif %}
{% if not worker.is_online %}
<i class="fas fa-exclamation-triangle text-danger blink" title="{% trans 'Last seen more than 10 minutes ago or never seen.' %}"></i>
{% endif %}
</td>
<td style="width: 1%; white-space: nowrap;">
{% if worker.workerstatus %}
{{ worker.workerstatus.config_version }}/{{ worker.workerstatus.dns_version }}
{% else %}
{% endif %}
</td>
<td style="width: 1%; white-space: nowrap;">
<a href="/cluster/worker/manage/?uuid={{ worker.uuid }}" title="{% trans 'Edit' %}"><i class="far fa-edit"></i></a>
</td>
</tr>
{% empty %}
<tr>
<td colspan="8" class="text-center text-muted">{% trans 'No workers configured' %}</td>
</tr>
{% endfor %}
</tbody>
</table>
<a href="/cluster/worker/manage/" class="btn btn-primary">{% trans 'Add Worker' %}</a>
<a href="/cluster/settings/" class="btn btn-secondary">{% trans 'Cluster Settings' %}</a>
<button id="btn-cluster-info" class="btn btn-outline-info float-right">{% trans 'Cluster Information' %}</button>
<div id="cluster-info-table" style="display: none; margin-top: 20px;">
<h4>{% trans 'Cluster Information' %}</h4>
<table class="table table-bordered" style="width: 50%;">
<tbody>
<tr>
<th>{% trans 'Enabled' %}</th>
<td>
{% if cluster_settings.enabled %}
<i class="fas fa-check text-green"></i>
{% else %}
<i class="fas fa-times text-gray"></i>
{% endif %}
</td>
</tr>
<tr>
<th>{% trans 'Cluster Mode' %}</th>
<td>{{ cluster_settings.get_cluster_mode_display }}</td>
</tr>
<tr>
<th>{% trans 'Config Version' %}</th>
<td>{{ cluster_settings.config_version }}</td>
</tr>
<tr>
<th>{% trans 'DNS Version' %}</th>
<td>{{ cluster_settings.dns_version }}</td>
</tr>
<tr>
<th>{% trans 'Stats Cache Refresh Interval' %}</th>
<td>{{ cache_refresh_interval }}s</td>
</tr>
<tr>
<th>{% trans 'Cache Enabled' %}</th>
<td>
{% if cache_enabled %}
<i class="fas fa-check text-green"></i>
{% else %}
<i class="fas fa-times text-gray"></i>
{% endif %}
</td>
</tr>
<tr>
<th>{% trans 'Cache Web Load Previous Count' %}</th>
<td>{{ cache_web_load_previous_count }}</td>
</tr>
<tr>
<th>{% trans 'Master Cache Processing Time' %}</th>
<td>{{ master_cache_times }}</td>
</tr>
<tr>
<th>{% trans 'Cluster Cache Processing Time' %}</th>
<td>{{ cluster_cache_times }}</td>
</tr>
<tr>
<th>{% trans 'Restart Mode' %}</th>
<td>{{ cluster_settings.get_restart_mode_display }}</td>
</tr>
<tr>
<th>{% trans 'Worker Display' %}</th>
<td>{{ cluster_settings.get_worker_display_display }}</td>
</tr>
<tr>
<th>{% trans 'Primary WireGuard' %}</th>
<td>
{% if cluster_settings.primary_enable_wireguard %}
<i class="fas fa-check text-green"></i>
{% else %}
<i class="fas fa-times text-gray"></i>
{% endif %}
</td>
</tr>
</tbody>
</table>
{% if not cluster_settings.enabled %}
<div class="alert alert-warning">
<i class="icon fas fa-exclamation-triangle"></i>
{% trans 'The cluster is disabled. Workers are refused and App Gateway applications are only published by this primary server.' %}
<a href="{% url 'cluster_settings' %}" class="alert-link">{% trans 'Cluster Settings' %}</a>
</div>
{% endif %}
{% if not cache_enabled %}
<div class="alert alert-danger">
<i class="icon fas fa-exclamation-circle"></i>
{% trans 'The cluster requires the WireGuard status cache. Enable WIREGUARD_STATUS_CACHE_ENABLED in the .env file.' %}
</div>
{% endif %}
<div class="modal fade" id="clusterFeatureModal" tabindex="-1" role="dialog" aria-labelledby="clusterFeatureModalLabel"
aria-hidden="true">
<div class="modal-dialog modal-lg" role="document">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title" id="clusterFeatureModalLabel">⚠️ {% trans "Cluster Feature – Testing Phase" %}
</h5>
<button type="button" class="close" data-dismiss="modal" aria-label="Close">
<span aria-hidden="true">&times;</span>
</button>
<div id="cluster-live">
<div class="row">
<div class="col-md-3 col-sm-6">
<div class="info-box">
<span class="info-box-icon {% if summary.online == summary.enabled %}bg-success{% else %}bg-danger{% endif %}"><i class="fas fa-server"></i></span>
<div class="info-box-content">
<span class="info-box-text">{% trans 'Workers online' %}</span>
<span class="info-box-number">{{ summary.online }} / {{ summary.enabled }}</span>
</div>
<div class="modal-body">
<p>{% trans "The cluster functionality is currently released as experimental." %}<br>
{% trans "At this stage, only core features are implemented:" %}</p>
<ul>
<li>{% trans "Configuration synchronization service" %}</li>
<li>{% trans "Primary node mirroring (WireGuard and firewall rules)" %}</li>
<li>{% trans "DNS rules mirroring" %}</li>
<li>{% trans "Traffic data synchronization" %}</li>
<li>{% trans "Server selection dropdown before downloading peer configs or VPN invites" %}</li>
</ul>
<hr>
<br>
<p>
{% trans "For more information, please visit:" %}
<a href="https://github.com/eduardogsilva/wireguard_webadmin/discussions/128" target="_blank">https://github.com/eduardogsilva/wireguard_webadmin/discussions/128</a></p>
</div>
</div>
<div class="col-md-3 col-sm-6">
<div class="info-box">
<span class="info-box-icon {% if summary.synced == summary.enabled %}bg-success{% else %}bg-info{% endif %}"><i class="fas fa-sync"></i></span>
<div class="info-box-content">
<span class="info-box-text">{% trans 'Synchronized' %}</span>
<span class="info-box-number">{{ summary.synced }} / {{ summary.enabled }}</span>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-primary" id="btn-dismiss-cluster-modal">{% trans "I Understand" %}</button>
</div>
</div>
<div class="col-md-3 col-sm-6">
<div class="info-box">
<span class="info-box-icon {% if summary.attention %}bg-warning{% else %}bg-success{% endif %}"><i class="fas fa-exclamation-triangle"></i></span>
<div class="info-box-content">
<span class="info-box-text">{% trans 'Need attention' %}</span>
<span class="info-box-number">{{ summary.attention }}</span>
</div>
</div>
</div>
<div class="col-md-3 col-sm-6">
<div class="info-box">
<span class="info-box-icon {% if cluster_settings.primary_enable_wireguard %}bg-success{% else %}bg-secondary{% endif %}"><i class="fas fa-network-wired"></i></span>
<div class="info-box-content">
<span class="info-box-text">{% trans 'Primary WireGuard' %}</span>
<span class="info-box-number">{% if cluster_settings.primary_enable_wireguard %}{% trans 'Enabled' %}{% else %}{% trans 'Disabled' %}{% endif %}</span>
</div>
</div>
</div>
</div>
<div class="card card-primary card-outline">
<div class="card-header">
<h3 class="card-title"><i class="fas fa-server"></i> {% trans 'Workers' %}</h3>
<div class="card-tools">
<a href="{% url 'worker_manage' %}" class="btn btn-sm btn-primary"><i class="fas fa-plus"></i> {% trans 'Add Worker' %}</a>
<a href="{% url 'cluster_settings' %}" class="btn btn-sm btn-secondary"><i class="fas fa-cog"></i> {% trans 'Cluster Settings' %}</a>
</div>
</div>
<div class="card-body p-0">
{% if workers %}
<div class="table-responsive">
<table class="table table-hover mb-0">
<thead>
<tr>
<th>{% trans 'Worker' %}</th>
<th>{% trans 'Status' %}</th>
<th>{% trans 'Synchronization' %}</th>
<th>{% trans 'Services' %}</th>
<th>{% trans 'Version' %}</th>
<th class="text-right">{% trans 'Actions' %}</th>
</tr>
</thead>
<tbody>
{% for worker in workers %}
<tr>
<td>
<a href="{% url 'worker_details' %}?uuid={{ worker.uuid }}"><strong>{{ worker.name }}</strong></a>
<br><small class="text-muted">
{{ worker.hostname|default:worker.ip_address|default:'-' }}
{% if worker.ip_lock %}<i class="fas fa-lock text-warning" title="{% trans 'IP Lock Enabled' %}"></i>{% endif %}
{% if worker.city or worker.country %} · {{ worker.city|default:'' }}{% if worker.city and worker.country %}, {% endif %}{{ worker.country|default:'' }}{% endif %}
</small>
</td>
<td class="text-nowrap">
{% if not worker.enabled %}
<span class="badge badge-secondary">{% trans 'Disabled' %}</span>
{% elif worker.error_status %}
<span class="badge badge-danger" title="{{ worker.get_error_status_display }}">{% trans 'Refused' %}</span>
{% elif worker.is_online %}
<span class="badge badge-success">{% trans 'Online' %}</span>
{% else %}
<span class="badge badge-danger">{% trans 'Offline' %}</span>
{% endif %}
<br><small class="text-muted" title="{% trans 'Last Seen' %}">
{% if worker.status %}{{ worker.status.last_seen|date:"SHORT_DATETIME_FORMAT" }}{% else %}{% trans 'Never' %}{% endif %}
</small>
</td>
<td>
{% if not worker.enabled %}
<span class="text-muted">-</span>
{% elif not worker.operational %}
<span class="badge badge-secondary" title="{% trans 'The worker is not receiving updates.' %}">{% trans 'Unknown' %}</span>
{% elif worker.sync_state.state == 'synced' %}
<span class="badge badge-success">{% trans 'Synchronized' %}</span>
{% elif worker.sync_state.state == 'error' %}
<span class="badge badge-danger" title="{{ worker.error_messages|join:' / ' }}">{% trans 'Error' %}</span>
<br><small class="text-muted">{{ worker.unsynced_sections|join:', ' }}</small>
{% else %}
<span class="badge badge-secondary">{% trans 'Pending' %}</span>
<br><small class="text-muted">{{ worker.unsynced_sections|join:', ' }}</small>
{% endif %}
</td>
<td>
{% for service, state in worker.services %}
{% if worker.operational %}
<span class="badge {% if state == 'running' %}badge-success{% else %}badge-danger{% endif %}">{{ service }}</span>
{% else %}
<span class="badge badge-secondary" title="{% trans 'Last reported state' %}: {{ state }}">{{ service }}</span>
{% endif %}
{% empty %}
<span class="text-muted">-</span>
{% endfor %}
{% if worker.operational and worker.warnings.certificate_issues %}
<br><small class="text-warning" title="{{ worker.warnings.certificate_issues|join:', ' }}">
<i class="fas fa-certificate"></i> {% trans 'Certificates unavailable or expiring soon' %}
</small>
{% endif %}
</td>
<td class="text-nowrap">
{% if worker.status %}
<span class="badge {% if worker.outdated %}badge-warning{% else %}badge-light{% endif %}"
{% if worker.outdated %}title="{% trans 'The worker is outdated. Please update it to the latest version.' %}"{% endif %}>
v{{ worker.status.worker_version }}
</span>
{% else %}
<span class="text-muted">-</span>
{% endif %}
</td>
<td class="text-right text-nowrap">
<a href="{% url 'worker_details' %}?uuid={{ worker.uuid }}" class="btn btn-sm btn-outline-info" title="{% trans 'Details' %}"><i class="fas fa-eye"></i></a>
<a href="{% url 'worker_manage' %}?uuid={{ worker.uuid }}" class="btn btn-sm btn-outline-secondary" title="{% trans 'Edit' %}"><i class="fas fa-edit"></i></a>
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
{% else %}
<div class="p-4 text-center text-muted">
<p>{% trans 'No workers configured' %}</p>
<a href="{% url 'worker_manage' %}" class="btn btn-primary"><i class="fas fa-plus"></i> {% trans 'Add Worker' %}</a>
</div>
{% endif %}
</div>
</div>
</div>
<div class="card card-secondary card-outline collapsed-card">
<div class="card-header">
<h3 class="card-title"><i class="fas fa-stethoscope"></i> {% trans 'Diagnostics' %}</h3>
<div class="card-tools">
<button type="button" class="btn btn-tool" data-card-widget="collapse"><i class="fas fa-plus"></i></button>
</div>
</div>
<div class="card-body">
<dl class="row mb-0">
<dt class="col-sm-4">{% trans 'Cluster Mode' %}</dt>
<dd class="col-sm-8">{{ cluster_settings.get_cluster_mode_display }}</dd>
<dt class="col-sm-4">{% trans 'Worker Display' %}</dt>
<dd class="col-sm-8">{{ cluster_settings.get_worker_display_display }}</dd>
<dt class="col-sm-4">{% trans 'Current worker version' %}</dt>
<dd class="col-sm-8">v{{ current_worker_version }}</dd>
<dt class="col-sm-4">{% trans 'Stats Cache Refresh Interval' %}</dt>
<dd class="col-sm-8">{{ cache_refresh_interval }}s</dd>
<dt class="col-sm-4">{% trans 'Cache Web Load Previous Count' %}</dt>
<dd class="col-sm-8">{{ cache_web_load_previous_count }}</dd>
<dt class="col-sm-4">{% trans 'Master Cache Processing Time' %}</dt>
<dd class="col-sm-8"><small>{{ master_cache_times|default:'-' }}</small></dd>
<dt class="col-sm-4">{% trans 'Cluster Cache Processing Time' %}</dt>
<dd class="col-sm-8"><small>{{ cluster_cache_times|default:'-' }}</small></dd>
</dl>
</div>
</div>
{% endblock %}
{% block custom_page_scripts %}
<script>
$(document).ready(function () {
// Cluster Info Button Logic
$('#btn-cluster-info').click(function () {
$('#cluster-info-table').slideToggle();
});
// Cluster Feature Modal Logic
if (!sessionStorage.getItem('cluster_modal_dismissed')) {
$('#clusterFeatureModal').modal('show');
}
// "I Understand" button persists dismissal
$('#btn-dismiss-cluster-modal').click(function () {
$('#clusterFeatureModal').modal('hide');
sessionStorage.setItem('cluster_modal_dismissed', 'true');
});
// Refresh the summary and the worker list every 30 seconds while the page is visible
setInterval(function () {
if (document.visibilityState === 'visible') {
$.get(window.location.href, function (data) {
var content = $('<div>').append($.parseHTML(data)).find('#cluster-live');
if (content.length) {
$('#cluster-live').html(content.html());
}
});
}
}, 30000);
});
</script>
{% endblock %}
{% endblock %}
@@ -1,4 +1,3 @@
import os
from django.conf import settings
from django.core.management.base import BaseCommand
@@ -21,10 +20,8 @@ class Command(BaseCommand):
if settings.CADDY_ENABLED:
self.stdout.write('Exporting Caddy configuration...')
try:
from app_gateway.caddy_config_export import export_caddy_config
export_caddy_config('/caddy_json_export/')
if settings.DEBUG:
export_caddy_config(os.path.join(settings.BASE_DIR, 'containers', 'caddy', 'config_files'))
from app_gateway.caddy_config_export import apply_caddy_config
apply_caddy_config()
self.stdout.write(self.style.SUCCESS('Caddy configuration exported.'))
except Exception as e:
self.stdout.write(self.style.WARNING(f'Failed to export Caddy configuration: {e}'))
+3 -15
View File
@@ -15,10 +15,9 @@ from django.shortcuts import Http404, get_object_or_404, redirect, render
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from cluster.models import ClusterSettings, Worker
from cluster.models import Worker
from firewall.models import RedirectRule, FirewallSettings
from firewall.tools import export_user_firewall, generate_firewall_footer, generate_firewall_header, \
generate_port_forward_firewall, generate_redirect_dns_rules, generate_route_policy_rules
from firewall.tools import generate_firewall_script
from user_manager.models import UserAcl
from vpn_invite.models import PeerInvite
from wgwadmlibrary.tools import user_has_access_to_peer
@@ -68,12 +67,7 @@ def generate_peer_config(peer_uuid, server_address=None):
def export_firewall_configuration():
firewall_content = generate_firewall_header()
firewall_content += generate_redirect_dns_rules()
firewall_content += generate_port_forward_firewall()
firewall_content += generate_route_policy_rules()
firewall_content += export_user_firewall()
firewall_content += generate_firewall_footer()
firewall_content = generate_firewall_script()
firewall_path = "/etc/wireguard/wg-firewall.sh"
with open(firewall_path, "w") as firewall_file:
firewall_file.write(firewall_content)
@@ -96,12 +90,6 @@ def set_instance_to_include_firewall():
def export_wireguard_configuration(instance_only: WireGuardInstance = None):
cluster_settings = ClusterSettings.objects.filter(name='cluster_settings', enabled=True).first()
if cluster_settings:
if WireGuardInstance.objects.filter(pending_changes=True).exists():
cluster_settings.config_version += 1
cluster_settings.save()
cleanup_orphaned = True
instance_to_execute_firewall, force_export_all_instances = set_instance_to_include_firewall()
+2 -2
View File
@@ -176,8 +176,8 @@ DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
WIREGUARD_WEBADMIN_VERSION = 9981
CLUSTER_WORKER_CURRENT_VERSION = 11
CLUSTER_WORKER_MINIMUM_VERSION = 11
CLUSTER_WORKER_CURRENT_VERSION = 12
CLUSTER_WORKER_MINIMUM_VERSION = 12
CADDY_ENABLED = os.getenv("CADDY_ENABLED", "false").lower() == "true"