diff --git a/api/views.py b/api/views.py index 41ed844..c5ca5d3 100644 --- a/api/views.py +++ b/api/views.py @@ -401,11 +401,21 @@ def func_concatenate_cluster_wireguard_status_cache() -> None: existing_peer = combined_data[interface][peer_key] + # Each server keeps its own counters, so the peer traffic is the sum of all sources + existing_transfer = existing_peer.get("transfer") or {} + new_transfer = peer_info_copy.get("transfer") or {} + combined_transfer = { + "tx": int(existing_transfer.get("tx", 0) or 0) + int(new_transfer.get("tx", 0) or 0), + "rx": int(existing_transfer.get("rx", 0) or 0) + int(new_transfer.get("rx", 0) or 0), + } + new_hs = _latest_handshake_as_int(peer_info_copy) old_hs = _latest_handshake_as_int(existing_peer) + # Handshake, endpoint and source come from the server where the peer connected last if new_hs > old_hs: combined_data[interface][peer_key] = peer_info_copy + combined_data[interface][peer_key]["transfer"] = combined_transfer processing_time_ms = int((time.monotonic() - start_time) * 1000) WireguardStatusCache.objects.create(data=combined_data, processing_time_ms=processing_time_ms, cache_type="cluster") diff --git a/app_gateway/caddy_config_export.py b/app_gateway/caddy_config_export.py index e557840..2efbcf1 100644 --- a/app_gateway/caddy_config_export.py +++ b/app_gateway/caddy_config_export.py @@ -1,11 +1,16 @@ +import hashlib import json import os +import shutil from django.conf import settings +from django.contrib import messages +from django.utils.translation import gettext as _ from app_gateway.models import ( AccessPolicy, Application, ApplicationHost, ApplicationPolicy, RESERVED_APP_NAME ) +from cluster.models import Worker from gatekeeper.models import ( AuthMethod, GatekeeperGroup, GatekeeperIPAddress, GatekeeperUser ) @@ -16,11 +21,43 @@ POLICY_TYPE_MAP = { 'deny': 'deny', } +# Written by the Caddy container of the primary server, read-only for Django +CERTIFICATE_EXPORT_DIR = '/caddy_cert_export' +ACME_CERTIFICATE_TYPES = ('http_challenge', 'dns_challenge') -def build_applications_data(): - applications = Application.objects.exclude(name=RESERVED_APP_NAME).prefetch_related('hosts') + +def get_served_applications(worker=None): + """Applications published by the primary server (worker=None) or by a cluster worker.""" + applications = Application.objects.prefetch_related('hosts', 'workers') + if worker is not None: + applications = applications.exclude(name=RESERVED_APP_NAME) + return [application for application in applications if application.is_served_by(worker)] + + +def certificate_file_name(certificate): + return hashlib.sha256(certificate.encode()).hexdigest() + + +def read_exported_certificate(hostname): + """Return (certificate, private_key) exported for a hostname, or (None, None).""" + if '/' in hostname or hostname in ('', '.', '..'): + return None, None + certificate_dir = os.path.join(CERTIFICATE_EXPORT_DIR, hostname) + try: + with open(os.path.join(certificate_dir, 'certificate.crt'), 'r', encoding='utf-8') as certificate_file: + certificate = certificate_file.read() + with open(os.path.join(certificate_dir, 'certificate.key'), 'r', encoding='utf-8') as key_file: + private_key = key_file.read() + except OSError: + return None, None + return certificate, private_key + + +def build_applications_data(worker=None): entries = [] - for app in applications: + for app in get_served_applications(worker): + if app.name == RESERVED_APP_NAME: + continue entry = { 'id': app.name, 'name': app.display_name or app.name, @@ -67,7 +104,8 @@ def _build_auth_method_entry(method): return entry -def build_auth_policies_data(): +def build_auth_policies_data(worker=None): + # Policies, methods, groups and users are referenced by name, so every server receives all of them auth_methods = {} for method in AuthMethod.objects.all(): auth_methods[method.name] = _build_auth_method_entry(method) @@ -102,13 +140,9 @@ def build_auth_policies_data(): } -def build_routes_data(): - applications = ( - Application.objects - .prefetch_related('routes__policy') - ) +def build_routes_data(worker=None): entries = {} - for app in applications: + for app in get_served_applications(worker): try: app_policy = ApplicationPolicy.objects.get(application=app) default_policy = app_policy.default_policy.name @@ -132,15 +166,41 @@ def build_routes_data(): return {'entries': entries} -def build_tls_data(): +def build_tls_data(worker=None): + """ + On the primary server every ACME certificate is issued, including the ones of applications that are only + served by cluster workers (certificate_only), and certificates used by workers are exported. Workers never + issue ACME certificates: they receive the exported certificate (certificate_file) or use the internal CA. + """ hosts = {} dns_credentials = {} - for application_host in ApplicationHost.objects.select_related('dns_credential'): + served_application_ids = {application.pk for application in get_served_applications(worker)} + + application_hosts = ApplicationHost.objects.select_related('dns_credential', 'application').prefetch_related( + 'application__workers' + ) + for application_host in application_hosts: + application = application_host.application + hostname = application_host.hostname.lower() + is_acme = application_host.certificate_type in ACME_CERTIFICATE_TYPES host_entry = { 'certificate_type': application_host.certificate_type, 'certificate_fallback': application_host.certificate_fallback, - 'certificate_generation': application_host.certificate_generation, } + + if worker is not None: + if application.pk in served_application_ids: + hosts[hostname] = host_entry + continue + + served_by_workers = application.name != RESERVED_APP_NAME and application.served_by_workers + if application.pk not in served_application_ids and not (served_by_workers and is_acme): + continue + host_entry['certificate_generation'] = application_host.certificate_generation + if served_by_workers and is_acme: + host_entry['export_certificate'] = True + if application.pk not in served_application_ids: + host_entry['certificate_only'] = True dns_credential = application_host.dns_credential if application_host.certificate_type == 'dns_challenge' and dns_credential: host_entry['dns_credential'] = dns_credential.name @@ -148,7 +208,7 @@ def build_tls_data(): 'provider': dns_credential.provider, 'credentials': dns_credential.credentials, } - hosts[application_host.hostname.lower()] = host_entry + hosts[hostname] = host_entry return { 'hosts': hosts, @@ -166,30 +226,98 @@ EXPORT_FILES = { } -def export_caddy_config(output_dir): - os.makedirs(output_dir, exist_ok=True) +# Cluster workers follow the configuration exported for them, so changes are only published by an export +WORKER_EXPORT_DIR_NAME = 'cluster' + +def get_worker_export_dir(output_dir, worker): + return os.path.join(output_dir, WORKER_EXPORT_DIR_NAME, str(worker.uuid)) + + +def _write_export_files(target_dir, worker=None): + os.makedirs(target_dir, exist_ok=True) for filename, builder in EXPORT_FILES.items(): - filepath = os.path.join(output_dir, filename) + filepath = os.path.join(target_dir, filename) with open(filepath, 'w', encoding='utf-8') as output_file: - json.dump(builder(), output_file, indent=2) + json.dump(builder(worker), output_file, indent=2) output_file.write('\n') -def has_pending_export(output_dir): - """Return True when the current configuration differs from the files last exported to output_dir.""" - for filename, builder in EXPORT_FILES.items(): - filepath = os.path.join(output_dir, filename) +def export_caddy_config(output_dir): + os.makedirs(output_dir, exist_ok=True) + workers = list(Worker.objects.all()) + for worker in workers: + _write_export_files(get_worker_export_dir(output_dir, worker), worker) + + workers_export_dir = os.path.join(output_dir, WORKER_EXPORT_DIR_NAME) + if os.path.isdir(workers_export_dir): + worker_uuids = {str(worker.uuid) for worker in workers} + for worker_dir_name in os.listdir(workers_export_dir): + if worker_dir_name not in worker_uuids: + shutil.rmtree(os.path.join(workers_export_dir, worker_dir_name), ignore_errors=True) + + # The primary server is written last: its files are the ones Caddy and the auth-gateway watch + _write_export_files(output_dir) + + +def load_exported_files(target_dir): + """Return {filename: data} from an export directory, or None when it is missing or incomplete.""" + exported_files = {} + for filename in EXPORT_FILES: try: - with open(filepath, 'r', encoding='utf-8') as exported_file: - exported_data = json.load(exported_file) + with open(os.path.join(target_dir, filename), 'r', encoding='utf-8') as exported_file: + exported_files[filename] = json.load(exported_file) except (OSError, ValueError): - return True - # Round-trip through JSON so the comparison matches what export_caddy_config writes - if json.loads(json.dumps(builder())) != exported_data: + return None + return exported_files + + +def load_worker_export(worker, output_dir=None): + return load_exported_files(get_worker_export_dir(output_dir or CADDY_EXPORT_DIR, worker)) + + +def add_certificate_files(tls_data): + """Point the ACME hosts of a worker to the certificates currently exported by the primary server.""" + for hostname, host_tls in tls_data.get('hosts', {}).items(): + if host_tls.get('certificate_type') not in ACME_CERTIFICATE_TYPES: + continue + certificate, private_key = read_exported_certificate(hostname) + host_tls['certificate_file'] = certificate_file_name(certificate) if certificate and private_key else '' + return tls_data + + +def _export_differs(target_dir, worker=None): + exported_files = load_exported_files(target_dir) + if exported_files is None: + return True + # Round-trip through JSON so the comparison matches what export_caddy_config writes + for filename, builder in EXPORT_FILES.items(): + if json.loads(json.dumps(builder(worker))) != exported_files[filename]: return True return False +def has_pending_export(output_dir): + """Return True when the current configuration differs from the files last exported to output_dir.""" + if _export_differs(output_dir): + return True + return any(_export_differs(get_worker_export_dir(output_dir, worker), worker) for worker in Worker.objects.all()) + + def caddy_export_pending(): return settings.CADDY_ENABLED and has_pending_export(CADDY_EXPORT_DIR) + + +def apply_caddy_config(): + """Export the configuration read by Caddy, the auth-gateway and the cluster workers.""" + export_caddy_config(CADDY_EXPORT_DIR) + if settings.DEBUG: + export_caddy_config(os.path.join(settings.BASE_DIR, 'containers', 'caddy', 'config_files')) + + +def auto_apply_caddy_config(request): + if not settings.AUTO_APPLY or not settings.CADDY_ENABLED: + return False + apply_caddy_config() + messages.info(request, _('Gateway configuration applied automatically.')) + return True diff --git a/app_gateway/forms.py b/app_gateway/forms.py index 7088da6..0a78fb7 100644 --- a/app_gateway/forms.py +++ b/app_gateway/forms.py @@ -12,22 +12,44 @@ from app_gateway.models import ( Application, ApplicationHost, AccessPolicy, ApplicationPolicy, ApplicationRoute, DnsChallengeCredential ) from app_gateway.setup_defaults import is_ip_address +from cluster.models import ClusterSettings, Worker class ApplicationForm(forms.ModelForm): class Meta: model = Application - fields = ['display_name', 'upstream', 'allow_invalid_cert'] + fields = ['display_name', 'upstream', 'allow_invalid_cert', 'serve_on_primary', 'all_workers', 'workers'] labels = { 'display_name': _('Name'), 'upstream': _('Upstream'), 'allow_invalid_cert': _('Allow invalid/self-signed certificate'), + 'serve_on_primary': _('Serve on the primary server'), + 'all_workers': _('Serve on all cluster workers'), + 'workers': _('Cluster workers'), } def __init__(self, *args, **kwargs): cancel_url = kwargs.pop('cancel_url', '#') super().__init__(*args, **kwargs) self.fields['display_name'].required = True + self.cluster_enabled = ClusterSettings.cluster_enabled() + + cluster_rows = [] + if self.cluster_enabled: + self.fields['workers'].queryset = Worker.objects.order_by('name') + self.fields['workers'].help_text = _('Select specific workers, or leave empty when serving on all cluster workers.') + cluster_rows = [ + HTML(f'
{_("Servers that publish this application.")}
'), + Div( + Div('serve_on_primary', css_class='col-md-12'), + Div('all_workers', css_class='col-md-12'), + Div('workers', css_class='col-md-12'), + css_class='row' + ), + ] + else: + for field_name in ('serve_on_primary', 'all_workers', 'workers'): + del self.fields[field_name] self.helper = FormHelper() self.helper.layout = Layout( @@ -40,6 +62,7 @@ class ApplicationForm(forms.ModelForm): Div('allow_invalid_cert', css_class='col-md-12'), css_class='row' ), + *cluster_rows, Div( Div( Submit('submit', _('Save'), css_class='btn btn-primary'), @@ -71,6 +94,22 @@ class ApplicationForm(forms.ModelForm): elif parsed.scheme == "http" and cleaned_data.get("allow_invalid_cert"): self.add_error("allow_invalid_cert", _("This option is only available for https:// upstreams.")) + if self.cluster_enabled: + all_workers = cleaned_data.get('all_workers') + selected_workers = cleaned_data.get('workers') + if all_workers and selected_workers: + self.add_error('workers', _('Leave this empty when the application is served on all cluster workers.')) + if not cleaned_data.get('serve_on_primary') and not all_workers and not selected_workers: + self.add_error(None, _('Select at least one server to publish this application.')) + served_by_workers = all_workers or bool(selected_workers) + if served_by_workers and self.instance.pk: + http_hosts = self.instance.hosts.filter(certificate_type='http_challenge').values_list('hostname', flat=True) + if http_hosts: + self.add_error(None, _( + 'Applications served by cluster workers require DNS validation or self-signed certificates. ' + 'Change the certificate of these hosts first: %(hosts)s' + ) % {'hosts': ', '.join(http_hosts)}) + return cleaned_data @@ -99,6 +138,12 @@ class ApplicationHostForm(forms.ModelForm): if hostname.lower() in webadmin_hostnames: self.add_error('hostname', _('This hostname is already used by WireGuard WebAdmin.')) + application = self.instance.application if self.instance.pk else self.application + if certificate_type == 'http_challenge' and application and application.served_by_workers: + self.add_error('certificate_type', _( + 'This application is served by cluster workers. Use DNS validation or a self-signed certificate.' + )) + if hostname and is_ip_address(hostname) and certificate_type != 'self_signed': self.add_error('certificate_type', _('IP addresses can only use a self-signed certificate.')) @@ -117,6 +162,7 @@ class ApplicationHostForm(forms.ModelForm): def __init__(self, *args, **kwargs): cancel_url = kwargs.pop('cancel_url', '#') self.is_reserved = kwargs.pop('is_reserved', False) + self.application = kwargs.pop('application', None) super().__init__(*args, **kwargs) self.fields['dns_credential'].required = False if self.is_reserved: diff --git a/app_gateway/management/commands/disable_webadmin_gateway_auth.py b/app_gateway/management/commands/disable_webadmin_gateway_auth.py index 9503ebc..4365165 100644 --- a/app_gateway/management/commands/disable_webadmin_gateway_auth.py +++ b/app_gateway/management/commands/disable_webadmin_gateway_auth.py @@ -1,11 +1,10 @@ -import os import sys from django.conf import settings from django.core.management.base import BaseCommand, CommandError from django.db import transaction -from app_gateway.caddy_config_export import export_caddy_config +from app_gateway.caddy_config_export import apply_caddy_config from app_gateway.models import AccessPolicy, Application, ApplicationPolicy, RESERVED_APP_NAME from app_gateway.setup_defaults import create_default_entries @@ -86,9 +85,7 @@ class Command(BaseCommand): if settings.CADDY_ENABLED: self.stdout.write('Exporting Caddy configuration...') - export_caddy_config('/caddy_json_export/') - if settings.DEBUG: - export_caddy_config(os.path.join(settings.BASE_DIR, 'containers', 'caddy', 'config_files')) + apply_caddy_config() self.stdout.write(self.style.SUCCESS('Caddy configuration exported.')) else: self.stdout.write('Skipping Caddy configuration export (CADDY_ENABLED is not set).') diff --git a/app_gateway/migrations/0013_application_served_by.py b/app_gateway/migrations/0013_application_served_by.py new file mode 100644 index 0000000..7ed78a9 --- /dev/null +++ b/app_gateway/migrations/0013_application_served_by.py @@ -0,0 +1,29 @@ +# Generated by Django 5.2.19 on 2026-11-19 16:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('app_gateway', '0012_certificate_type_labels'), + ('cluster', '0018_cluster_v2'), + ] + + operations = [ + migrations.AddField( + model_name='application', + name='all_workers', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='application', + name='serve_on_primary', + field=models.BooleanField(default=True), + ), + migrations.AddField( + model_name='application', + name='workers', + field=models.ManyToManyField(blank=True, related_name='applications', to='cluster.worker'), + ), + ] diff --git a/app_gateway/models.py b/app_gateway/models.py index faebaa6..1c16613 100644 --- a/app_gateway/models.py +++ b/app_gateway/models.py @@ -4,6 +4,7 @@ from django.db import models from django.utils.translation import gettext_lazy as _ from app_gateway.dns_providers import DNS_PROVIDER_CHOICES +from cluster.models import ClusterSettings from gatekeeper.models import GatekeeperGroup, AuthMethod, _unique_slug RESERVED_APP_NAME = 'wireguard_webadmin' @@ -44,6 +45,9 @@ class Application(models.Model): display_name = models.CharField(max_length=128, blank=True) upstream = models.CharField(max_length=255, help_text=_("Upstream address, e.g.: http://10.188.18.27:3000")) allow_invalid_cert = models.BooleanField(default=False, help_text=_("Allow invalid or self-signed TLS certificates from the upstream")) + serve_on_primary = models.BooleanField(default=True) + all_workers = models.BooleanField(default=False) + workers = models.ManyToManyField('cluster.Worker', blank=True, related_name='applications') created = models.DateTimeField(auto_now_add=True) updated = models.DateTimeField(auto_now=True) @@ -59,6 +63,18 @@ class Application(models.Model): def __str__(self): return self.display_name or self.name + # Without an active cluster every application is served by the primary server only + @property + def served_by_workers(self): + return ClusterSettings.cluster_enabled() and (self.all_workers or self.workers.exists()) + + def is_served_by(self, worker): + if not ClusterSettings.cluster_enabled(): + return worker is None + if worker is None: + return self.serve_on_primary + return self.all_workers or self.workers.filter(pk=worker.pk).exists() + class Meta: ordering = ['name'] diff --git a/app_gateway/tests.py b/app_gateway/tests.py index aa99520..8efd6f4 100644 --- a/app_gateway/tests.py +++ b/app_gateway/tests.py @@ -8,13 +8,20 @@ from pathlib import Path from unittest.mock import patch from django.conf import settings -from django.test import TestCase +from django.contrib.auth.models import User +from django.test import TestCase, override_settings from app_gateway.caddy_config_export import export_caddy_config -from app_gateway.models import AccessPolicy, Application, ApplicationHost, ApplicationPolicy, ApplicationRoute +from app_gateway.forms import ApplicationForm, ApplicationHostForm +from app_gateway.models import ( + AccessPolicy, Application, ApplicationHost, ApplicationPolicy, ApplicationRoute, DnsChallengeCredential +) +from app_gateway.setup_defaults import create_default_entries +from cluster.models import ClusterSettings, Worker from gatekeeper.models import ( AuthMethod, AuthMethodAllowedDomain, AuthMethodAllowedEmail, GatekeeperGroup, GatekeeperIPAddress, GatekeeperUser ) +from user_manager.models import UserAcl AUTH_GATEWAY_FIXTURES_DIR = Path(settings.BASE_DIR) / 'containers' / 'auth-gateway' / 'tests' / 'fixtures' / 'config_files' UPDATE_FIXTURES_ENV = 'UPDATE_AUTH_GATEWAY_FIXTURES' @@ -29,6 +36,11 @@ _WEBADMIN_EXPORT_SPEC = importlib.util.spec_from_file_location('caddy_export_wir _WEBADMIN_EXPORT_MODULE = importlib.util.module_from_spec(_WEBADMIN_EXPORT_SPEC) _WEBADMIN_EXPORT_SPEC.loader.exec_module(_WEBADMIN_EXPORT_MODULE) +_PROCESS_CONFIG_PATH = Path(settings.BASE_DIR) / 'containers' / 'caddy' / 'process_config.py' +_PROCESS_CONFIG_SPEC = importlib.util.spec_from_file_location('caddy_process_config', _PROCESS_CONFIG_PATH) +process_config = importlib.util.module_from_spec(_PROCESS_CONFIG_SPEC) +_PROCESS_CONFIG_SPEC.loader.exec_module(process_config) + def create_auth_gateway_scenario(): local_password = AuthMethod.objects.create(display_name='Local Password', auth_type='local_password') @@ -106,10 +118,16 @@ def export_auth_gateway_fixtures(output_dir): _WEBADMIN_EXPORT_MODULE._write_config(os.path.join(output_dir, 'wireguard_webadmin.json'), webadmin_config) +# Same hosts used for wireguard_webadmin.json, so the WireGuard WebAdmin hosts do not depend on the local .env +FIXTURE_ALLOWED_HOSTS = ['wireguard-webadmin', 'webadmin.example.test', 'webadmin2.example.test', '10.0.0.1'] + + +@override_settings(ALLOWED_HOSTS=FIXTURE_ALLOWED_HOSTS) class AuthGatewayFixturesTests(TestCase): maxDiff = None def setUp(self): + create_default_entries() create_auth_gateway_scenario() def test_auth_gateway_fixtures_match_exporter(self): @@ -131,3 +149,289 @@ class AuthGatewayFixturesTests(TestCase): f"{filename} is out of date. Regenerate it with: " f"{UPDATE_FIXTURES_ENV}=1 python manage.py test app_gateway" ) + + +CLOUDFLARE_TOKEN = 'cfut_' + 'a' * 40 + + +def caddy_app(app_id, hosts): + return {'id': app_id, 'hosts': hosts, 'upstream': 'http://10.0.0.10:8080'} + + +def site_block(caddyfile, host): + """Return the lines of the site block that starts with the given host.""" + lines = caddyfile.splitlines() + start = next(index for index, line in enumerate(lines) if line.startswith(host) and line.endswith('{')) + end = next(index for index in range(start + 1, len(lines)) if lines[index] == '}') + return lines[start:end + 1] + + +class CaddyfileTlsTests(TestCase): + tls_data = { + 'hosts': { + 'http.example.test': {'certificate_type': 'http_challenge'}, + 'strict.example.test': {'certificate_type': 'http_challenge', 'certificate_fallback': False}, + 'dns.example.test': {'certificate_type': 'dns_challenge', 'dns_credential': 'cloudflare'}, + 'internal.example.test': {'certificate_type': 'self_signed'}, + 'worker-only.example.test': {'certificate_type': 'dns_challenge', 'dns_credential': 'cloudflare', + 'certificate_only': True, 'export_certificate': True}, + }, + 'dns_credentials': {'cloudflare': {'provider': 'cloudflare', 'credentials': {'api_token': CLOUDFLARE_TOKEN}}}, + } + + def build(self, apps, tls_data=None): + return process_config.build_caddyfile(apps, None, None, tls_data or self.tls_data) + + def test_hosts_are_grouped_by_certificate_settings(self): + caddyfile = self.build([caddy_app('app', [ + 'http.example.test', 'strict.example.test', 'dns.example.test', 'internal.example.test', 'unknown.example.test' + ])]) + self.assertEqual( + site_block(caddyfile, 'http.example.test')[:5], + ['http.example.test, unknown.example.test {', ' tls {', ' issuer acme', ' issuer internal', ' }'], + ) + self.assertEqual(site_block(caddyfile, 'strict.example.test')[:4], ['strict.example.test {', ' tls {', ' issuer acme', ' }']) + self.assertEqual(site_block(caddyfile, 'internal.example.test')[1], ' tls internal') + + dns_block = '\n'.join(site_block(caddyfile, 'dns.example.test')) + self.assertIn(f'api_token "{CLOUDFLARE_TOKEN}"', dns_block) + self.assertIn('propagation_delay 30s', dns_block) + self.assertIn('propagation_timeout -1', dns_block) + self.assertIn('issuer internal', dns_block) + + def test_dns_challenge_can_be_disabled_after_invalid_configuration(self): + with patch.object(process_config, 'DISABLE_DNS_CHALLENGE', True): + caddyfile = self.build([caddy_app('app', ['dns.example.test'])]) + self.assertNotIn('dns cloudflare', caddyfile) + self.assertIn('issuer acme', '\n'.join(site_block(caddyfile, 'dns.example.test'))) + + def test_primary_keeps_certificates_of_worker_only_hosts(self): + caddyfile = self.build([caddy_app('app', ['http.example.test'])]) + worker_only_block = site_block(caddyfile, 'worker-only.example.test') + self.assertIn(' respond 404', worker_only_block) + self.assertIn(' dns cloudflare {', worker_only_block) + + def test_cluster_node_never_requests_acme_certificates(self): + node_tls = {'hosts': { + 'ready.example.test': {'certificate_type': 'dns_challenge', 'certificate_file': 'a' * 64}, + 'waiting.example.test': {'certificate_type': 'dns_challenge', 'certificate_file': ''}, + 'strict.example.test': {'certificate_type': 'dns_challenge', 'certificate_fallback': False, 'certificate_file': ''}, + 'internal.example.test': {'certificate_type': 'self_signed'}, + }, 'dns_credentials': {}} + apps = [caddy_app('app', list(node_tls['hosts']))] + with patch.object(process_config, 'CLUSTER_NODE', True), \ + patch.object(process_config, 'NODE_CERTIFICATES_DIR', '/caddy_certificates'): + caddyfile = self.build(apps, node_tls) + + self.assertNotIn('issuer acme', caddyfile) + self.assertNotIn('strict.example.test', caddyfile) + certificate_path = '/caddy_certificates/' + 'a' * 64 + self.assertEqual(site_block(caddyfile, 'ready.example.test')[1], f' tls {certificate_path}.crt {certificate_path}.key') + self.assertEqual(site_block(caddyfile, 'waiting.example.test')[0], 'waiting.example.test, internal.example.test {') + self.assertEqual(site_block(caddyfile, 'waiting.example.test')[1], ' tls internal') + + +class CaddyTlsStateTests(TestCase): + """Fallback certificates are removed and Caddy is restarted only when the certificate settings change.""" + + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + self.addCleanup(self.temp_dir.cleanup) + self.certificates_dir = os.path.join(self.temp_dir.name, 'certificates') + self.json_dir = os.path.join(self.temp_dir.name, 'json') + self.export_dir = os.path.join(self.temp_dir.name, 'export') + for directory in (self.certificates_dir, self.json_dir, self.export_dir): + os.makedirs(directory) + for attribute, value in ( + ('CERTIFICATES_DIR', self.certificates_dir), + ('TLS_STATE_PATH', os.path.join(self.temp_dir.name, 'tls_state.json')), + ('TLS_STATE_PENDING_PATH', os.path.join(self.temp_dir.name, 'tls_state_pending.json')), + ('JSON_DIR', self.json_dir), + ('CERTIFICATE_EXPORT_DIR', self.export_dir), + ): + patcher = patch.object(process_config, attribute, value) + patcher.start() + self.addCleanup(patcher.stop) + + def store_certificate(self, issuer_dir_name, host, content='certificate'): + host_dir = os.path.join(self.certificates_dir, issuer_dir_name, host) + os.makedirs(host_dir, exist_ok=True) + for extension in ('crt', 'key'): + with open(os.path.join(host_dir, f'{host}.{extension}'), 'w', encoding='utf-8') as certificate_file: + certificate_file.write(f'{content}-{extension}') + return host_dir + + def apply(self, host_tls): + apps = [caddy_app('app', ['app.example.test'])] + tls_data = {'hosts': {'app.example.test': host_tls}, 'dns_credentials': {}} + process_config._write_json(process_config.TLS_STATE_PENDING_PATH, process_config.build_tls_state(apps, tls_data)) + output = io.StringIO() + with contextlib.redirect_stdout(output), contextlib.redirect_stderr(io.StringIO()): + process_config.sync_tls_state() + return output.getvalue().strip() == 'restart' + + def test_first_run_never_restarts(self): + fallback_dir = self.store_certificate('local', 'app.example.test') + self.assertFalse(self.apply({'certificate_type': 'http_challenge'})) + self.assertTrue(os.path.isdir(fallback_dir)) + + def test_retry_removes_fallback_certificate(self): + self.apply({'certificate_type': 'http_challenge'}) + fallback_dir = self.store_certificate('local', 'app.example.test') + self.assertTrue(self.apply({'certificate_type': 'http_challenge', 'certificate_generation': 1})) + self.assertFalse(os.path.isdir(fallback_dir)) + + def test_retry_keeps_valid_acme_certificate_without_restart(self): + self.apply({'certificate_type': 'http_challenge'}) + acme_dir = self.store_certificate('acme-v02.api.letsencrypt.org-directory', 'app.example.test') + self.assertFalse(self.apply({'certificate_type': 'http_challenge', 'certificate_generation': 1})) + self.assertTrue(os.path.isdir(acme_dir)) + + def test_switching_to_self_signed_restarts_when_acme_certificate_is_cached(self): + self.apply({'certificate_type': 'http_challenge'}) + self.store_certificate('acme-v02.api.letsencrypt.org-directory', 'app.example.test') + self.assertTrue(self.apply({'certificate_type': 'self_signed'})) + + def test_unchanged_settings_do_not_restart(self): + self.apply({'certificate_type': 'http_challenge'}) + self.store_certificate('local', 'app.example.test') + self.assertFalse(self.apply({'certificate_type': 'http_challenge'})) + + def test_readded_host_with_leftover_fallback_is_retried(self): + self.apply({'certificate_type': 'http_challenge'}) + fallback_dir = self.store_certificate('local', 'other.example.test') + apps = [caddy_app('app', ['app.example.test', 'other.example.test'])] + tls_data = {'hosts': {'app.example.test': {'certificate_type': 'http_challenge'}, + 'other.example.test': {'certificate_type': 'dns_challenge', 'dns_credential': 'cloudflare'}}, + 'dns_credentials': {'cloudflare': {'provider': 'cloudflare', 'credentials': {'api_token': CLOUDFLARE_TOKEN}}}} + process_config._write_json(process_config.TLS_STATE_PENDING_PATH, process_config.build_tls_state(apps, tls_data)) + output = io.StringIO() + with contextlib.redirect_stdout(output), contextlib.redirect_stderr(io.StringIO()): + process_config.sync_tls_state() + self.assertEqual(output.getvalue().strip(), 'restart') + self.assertFalse(os.path.isdir(fallback_dir)) + + def test_new_host_without_cached_certificate_does_not_restart(self): + self.apply({'certificate_type': 'http_challenge'}) + apps = [caddy_app('app', ['app.example.test', 'new.example.test'])] + tls_data = {'hosts': {'app.example.test': {'certificate_type': 'http_challenge'}, + 'new.example.test': {'certificate_type': 'http_challenge'}}, 'dns_credentials': {}} + process_config._write_json(process_config.TLS_STATE_PENDING_PATH, process_config.build_tls_state(apps, tls_data)) + output = io.StringIO() + with contextlib.redirect_stdout(output), contextlib.redirect_stderr(io.StringIO()): + process_config.sync_tls_state() + self.assertEqual(output.getvalue().strip(), '') + + def test_cluster_node_restarts_when_certificate_changes(self): + with patch.object(process_config, 'CLUSTER_NODE', True): + self.apply({'certificate_type': 'dns_challenge', 'certificate_file': ''}) + fallback_dir = self.store_certificate('local', 'app.example.test') + self.assertTrue(self.apply({'certificate_type': 'dns_challenge', 'certificate_file': 'a' * 64})) + self.assertFalse(os.path.isdir(fallback_dir)) + self.assertTrue(self.apply({'certificate_type': 'dns_challenge', 'certificate_file': 'b' * 64})) + + def test_only_acme_certificates_are_exported(self): + tls_data = {'hosts': { + 'exported.example.test': {'certificate_type': 'dns_challenge', 'export_certificate': True}, + 'fallback.example.test': {'certificate_type': 'dns_challenge', 'export_certificate': True}, + 'local.example.test': {'certificate_type': 'dns_challenge'}, + }} + with open(os.path.join(self.json_dir, 'tls.json'), 'w', encoding='utf-8') as tls_file: + json.dump(tls_data, tls_file) + self.store_certificate('acme-v02.api.letsencrypt.org-directory', 'exported.example.test', 'acme') + self.store_certificate('acme-v02.api.letsencrypt.org-directory', 'local.example.test', 'acme') + self.store_certificate('local', 'fallback.example.test', 'internal') + os.makedirs(os.path.join(self.export_dir, 'removed.example.test')) + + with contextlib.redirect_stdout(io.StringIO()): + process_config.export_certificates() + + self.assertEqual(sorted(os.listdir(self.export_dir)), ['exported.example.test']) + with open(os.path.join(self.export_dir, 'exported.example.test', 'certificate.crt'), encoding='utf-8') as certificate_file: + self.assertEqual(certificate_file.read(), 'acme-crt') + + +@override_settings(ALLOWED_HOSTS=FIXTURE_ALLOWED_HOSTS) +class ClusterApplicationFormTests(TestCase): + def setUp(self): + ClusterSettings.objects.update_or_create(name='cluster_settings', defaults={'enabled': True}) + self.worker = Worker.objects.create(name='node-a') + self.application = Application.objects.create(display_name='Wiki', upstream='http://10.0.0.30:8080') + self.credential = DnsChallengeCredential.objects.create( + display_name='Cloudflare', provider='cloudflare', credentials={'api_token': CLOUDFLARE_TOKEN} + ) + + def test_application_with_http_validation_hosts_cannot_be_served_by_workers(self): + ApplicationHost.objects.create(application=self.application, hostname='wiki.example.test') + form = ApplicationForm(data={ + 'display_name': 'Wiki', 'upstream': 'http://10.0.0.30:8080', 'serve_on_primary': 'on', + 'workers': [self.worker.pk], + }, instance=self.application) + self.assertFalse(form.is_valid()) + self.assertIn('wiki.example.test', str(form.errors)) + + def test_all_workers_and_specific_workers_are_exclusive(self): + form = ApplicationForm(data={ + 'display_name': 'Wiki', 'upstream': 'http://10.0.0.30:8080', 'serve_on_primary': 'on', + 'all_workers': 'on', 'workers': [self.worker.pk], + }, instance=self.application) + self.assertIn('workers', form.errors) + + def test_application_must_be_published_somewhere(self): + form = ApplicationForm(data={'display_name': 'Wiki', 'upstream': 'http://10.0.0.30:8080'}, instance=self.application) + self.assertFalse(form.is_valid()) + self.assertIn('__all__', form.errors) + + def test_without_cluster_everything_is_served_by_the_primary(self): + self.application.serve_on_primary = False + self.application.save() + self.application.workers.add(self.worker) + ClusterSettings.objects.filter(name='cluster_settings').update(enabled=False) + self.assertTrue(self.application.is_served_by(None)) + self.assertFalse(self.application.is_served_by(self.worker)) + self.assertFalse(self.application.served_by_workers) + form = ApplicationForm(data={'display_name': 'Wiki', 'upstream': 'http://10.0.0.30:8080'}, instance=self.application) + self.assertTrue(form.is_valid(), form.errors) + self.assertNotIn('workers', form.fields) + + def test_host_of_application_served_by_workers_requires_dns_or_self_signed(self): + self.application.workers.add(self.worker) + form = ApplicationHostForm( + data={'hostname': 'wiki.example.test', 'certificate_type': 'http_challenge', 'certificate_fallback': 'on'}, + application=self.application, + ) + self.assertIn('certificate_type', form.errors) + form = ApplicationHostForm(data={ + 'hostname': 'wiki.example.test', 'certificate_type': 'dns_challenge', + 'dns_credential': self.credential.pk, 'certificate_fallback': 'on', + }, application=self.application) + self.assertTrue(form.is_valid(), form.errors) + + +@override_settings(ALLOWED_HOSTS=FIXTURE_ALLOWED_HOSTS + ['testserver'], CADDY_ENABLED=True, DEBUG=False) +class GatewayAutoApplyTests(TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + self.addCleanup(self.temp_dir.cleanup) + patcher = patch('app_gateway.caddy_config_export.CADDY_EXPORT_DIR', self.temp_dir.name) + patcher.start() + self.addCleanup(patcher.stop) + user = User.objects.create_user(username='gateway-admin', password='unused-test-password') + UserAcl.objects.create(user=user, user_level=50) + self.client.force_login(user) + + def save_application(self): + return self.client.post('/app_gateway/application/manage/', { + 'display_name': 'Auto Apply', 'upstream': 'http://10.0.0.40:8080', + }) + + @override_settings(AUTO_APPLY=True) + def test_saving_applies_configuration(self): + self.save_application() + with open(os.path.join(self.temp_dir.name, 'applications.json'), encoding='utf-8') as applications_file: + self.assertIn('auto-apply', [entry['id'] for entry in json.load(applications_file)['entries']]) + + @override_settings(AUTO_APPLY=False) + def test_disabled_auto_apply_keeps_changes_pending(self): + self.save_application() + self.assertFalse(os.path.exists(os.path.join(self.temp_dir.name, 'applications.json'))) diff --git a/app_gateway/views.py b/app_gateway/views.py index 9e58820..ec736f0 100644 --- a/app_gateway/views.py +++ b/app_gateway/views.py @@ -1,4 +1,4 @@ -import os +from datetime import timedelta from django.conf import settings from django.contrib import messages @@ -7,13 +7,18 @@ from django.db.models import F, ProtectedError from django.http import JsonResponse from django.shortcuts import render, get_object_or_404, redirect from django.urls import reverse +from django.utils import timezone +from django.utils.dateparse import parse_datetime from django.utils.formats import date_format from django.utils.http import url_has_allowed_host_and_scheme from django.utils.timezone import localtime from django.utils.translation import gettext as _ from django.views.decorators.http import require_POST -from app_gateway.caddy_config_export import CADDY_EXPORT_DIR, caddy_export_pending, export_caddy_config +from app_gateway.caddy_config_export import ( + ACME_CERTIFICATE_TYPES, CADDY_EXPORT_DIR, apply_caddy_config, auto_apply_caddy_config, caddy_export_pending, + load_exported_files, load_worker_export +) from app_gateway.certificate_status import inspect_certificate from app_gateway.forms import ( ApplicationForm, ApplicationHostForm, AccessPolicyForm, @@ -24,6 +29,7 @@ from app_gateway.models import ( RESERVED_APP_NAME ) from app_gateway.setup_defaults import create_default_entries +from cluster.models import ClusterSettings, Worker from user_manager.models import UserAcl @@ -79,6 +85,36 @@ def view_application_details(request): return render(request, 'app_gateway/application_details.html', context) +def worker_runs_gateway(worker): + # Workers running the App Gateway report the Caddy service; VPN-only workers do not serve applications + worker_status = getattr(worker, 'workerstatus', None) + return bool(worker_status and 'caddy' in (worker_status.services_status or {})) + + +def describe_certificate_status(certificate, certificate_type): + """Badge information for a certificate inspected on the primary server or reported by a cluster worker.""" + expiration = '' + if certificate.get('not_after'): + not_after = certificate['not_after'] + if isinstance(not_after, str): + not_after = parse_datetime(not_after) + if not_after: + expiration = _('expires %(date)s') % {'date': date_format(localtime(not_after), 'SHORT_DATETIME_FORMAT')} + + if certificate.get('status') == 'trusted': + return 'success', _('Trusted'), f"{certificate.get('issuer', '')}, {expiration}" + if certificate.get('status') == 'untrusted' and certificate_type == 'self_signed': + return 'secondary', _('Self-signed'), expiration + if certificate.get('status') == 'untrusted': + detail = _('The trusted certificate could not be issued yet, %(expiration)s') % {'expiration': expiration} + return 'warning', _('Using fallback'), detail + if certificate.get('status') == 'unknown': + return 'light', _('Unknown'), _('No recent report from this server.') + if certificate.get('status') == 'pending_export': + return 'info', _('Pending export'), _('Export the configuration to publish this host on this server.') + return 'danger', _('Unavailable'), _('No certificate is being served for this hostname.') + + @login_required def view_application_certificate_status(request): if not UserAcl.objects.filter(user=request.user).filter(user_level__gte=20).exists(): @@ -88,29 +124,53 @@ def view_application_certificate_status(request): if not settings.CADDY_ENABLED: return JsonResponse({'caddy_enabled': False, 'hosts': {}}) + workers = [] + if application.name != RESERVED_APP_NAME and ClusterSettings.objects.filter(name='cluster_settings', enabled=True).exists(): + workers = [ + worker for worker in Worker.objects.filter(enabled=True).select_related('workerstatus').order_by('name') + if application.is_served_by(worker) and worker_runs_gateway(worker) + ] + report_limit = timezone.now() - timedelta(minutes=10) + + # Servers only publish what was exported, so hosts missing from their exported files are pending + primary_export = load_exported_files(CADDY_EXPORT_DIR) or {} + primary_exported_hosts = primary_export.get('tls.json', {}).get('hosts', {}) + worker_exported_hosts = {} + for worker in workers: + worker_export = load_worker_export(worker) or {} + worker_exported_hosts[worker.pk] = worker_export.get('tls.json', {}).get('hosts', {}) + hosts_status = {} for application_host in application.hosts.all(): - certificate = inspect_certificate(application_host.hostname) - expiration = '' - if certificate['not_after']: - expiration = _('expires %(date)s') % { - 'date': date_format(localtime(certificate['not_after']), 'SHORT_DATETIME_FORMAT') - } + hostname = application_host.hostname.lower() + servers = [] + # The primary server also manages ACME certificates of applications only served by workers + primary_handles_host = application.is_served_by(None) or ( + application_host.certificate_type in ACME_CERTIFICATE_TYPES and application.served_by_workers + ) + if primary_handles_host: + if hostname in primary_exported_hosts: + certificate = inspect_certificate(application_host.hostname) + else: + certificate = {'status': 'pending_export'} + servers.append((_('Primary Server'), certificate)) - if certificate['status'] == 'trusted': - css_class, label = 'success', _('Trusted') - detail = f"{certificate['issuer']}, {expiration}" - elif certificate['status'] == 'untrusted' and application_host.certificate_type == 'self_signed': - css_class, label = 'secondary', _('Self-signed') - detail = expiration - elif certificate['status'] == 'untrusted': - css_class, label = 'warning', _('Using fallback') - detail = _('The trusted certificate could not be issued yet, %(expiration)s') % {'expiration': expiration} - else: - css_class, label = 'danger', _('Unavailable') - detail = _('No certificate is being served for this hostname.') + for worker in workers: + worker_status = getattr(worker, 'workerstatus', None) + if hostname not in worker_exported_hosts[worker.pk]: + certificate = {'status': 'pending_export'} + elif worker_status and worker_status.certificate_status_updated and worker_status.certificate_status_updated > report_limit: + certificate = worker_status.certificate_status.get(hostname, {'status': 'unavailable'}) + else: + certificate = {'status': 'unknown'} + servers.append((worker.display_name, certificate)) - hosts_status[str(application_host.uuid)] = {'css_class': css_class, 'label': label, 'detail': detail} + hosts_status[str(application_host.uuid)] = [] + for server_name, certificate in servers: + css_class, label, detail = describe_certificate_status(certificate, application_host.certificate_type) + hosts_status[str(application_host.uuid)].append({ + 'server': server_name if workers else '', 'css_class': css_class, 'label': label, 'detail': detail, + }) return JsonResponse({'caddy_enabled': True, 'hosts': hosts_status}) @@ -132,13 +192,17 @@ def view_manage_application(request): application = None title = _('Create Application') - cancel_url = reverse('app_gateway_list') + '?tab=applications' + if application: + cancel_url = reverse('view_application') + f'?uuid={application.uuid}' + else: + cancel_url = reverse('app_gateway_list') + '?tab=applications' form = ApplicationForm(request.POST or None, instance=application, cancel_url=cancel_url) if form.is_valid(): - form.save() + application = form.save() messages.success(request, _('Application saved successfully.')) - return redirect(cancel_url) + auto_apply_caddy_config(request) + return redirect(reverse('view_application') + f'?uuid={application.uuid}') form_description = { 'size': 'col-lg-6', @@ -178,6 +242,7 @@ def view_delete_application(request): if request.method == 'POST': application.delete() messages.success(request, _('Application deleted successfully.')) + auto_apply_caddy_config(request) return redirect(cancel_url) context = { @@ -214,13 +279,15 @@ def view_manage_application_host(request): return redirect(cancel_url) form = ApplicationHostForm( - request.POST or None, instance=application_host, cancel_url=cancel_url, is_reserved=is_reserved + request.POST or None, instance=application_host, cancel_url=cancel_url, is_reserved=is_reserved, + application=application, ) if form.is_valid(): host = form.save(commit=False) host.application = application host.save() messages.success(request, _('Application Host saved successfully.')) + auto_apply_caddy_config(request) return redirect(cancel_url) form_description_content = _(''' @@ -233,6 +300,7 @@ def view_manage_application_host(request):When the fallback option is enabled and a trusted certificate cannot be issued, a self-signed certificate is used until the next successful attempt. Without the fallback, the connection fails until the certificate is issued.
Use Retry certificate in the host list to request a new attempt right away, for example after fixing the DNS or firewall.
+Important: the certificate is requested as soon as the configuration is applied. With HTTP validation, the hostname must already point to this server: failed attempts count towards the Let's Encrypt limit (5 per hour per hostname). If the DNS is not ready yet, use DNS validation.
''') if is_reserved: form_description_content += _(''' @@ -268,6 +336,7 @@ def view_delete_application_host(request): if request.method == 'POST': application_host.delete() messages.success(request, _('Application Host deleted successfully.')) + auto_apply_caddy_config(request) return redirect(cancel_url) context = { @@ -312,6 +381,7 @@ def view_manage_access_policy(request): if form.is_valid(): form.save() messages.success(request, _('Access Policy saved successfully.')) + auto_apply_caddy_config(request) return redirect(cancel_url) if policy_type == 'public': @@ -365,6 +435,7 @@ def view_delete_access_policy(request): try: access_policy.delete() messages.success(request, _('Access Policy deleted successfully.')) + auto_apply_caddy_config(request) except ProtectedError: messages.error(request, _('Cannot delete this Access Policy because it is currently in use by an Application Route or Application Default Policy.')) return redirect(cancel_url) @@ -403,6 +474,7 @@ def view_manage_application_policy(request): policy_config.application = application policy_config.save() messages.success(request, _('Application Default Policy saved successfully.')) + auto_apply_caddy_config(request) return redirect(cancel_url) context = { @@ -426,6 +498,7 @@ def view_delete_application_policy(request): if request.method == 'POST': application_policy.delete() messages.success(request, _('Application Default Policy deleted successfully.')) + auto_apply_caddy_config(request) return redirect(cancel_url) context = { @@ -464,6 +537,7 @@ def view_manage_application_route(request): route.application = application route.save() messages.success(request, _('Application Route saved successfully.')) + auto_apply_caddy_config(request) return redirect(cancel_url) form_description = { @@ -502,6 +576,7 @@ def view_delete_application_route(request): if request.method == 'POST': application_route.delete() messages.success(request, _('Application Route deleted successfully.')) + auto_apply_caddy_config(request) return redirect(cancel_url) context = { @@ -534,9 +609,7 @@ def view_retry_application_host_certificate(request): return redirect(redirect_url) ApplicationHost.objects.filter(pk=application_host.pk).update(certificate_generation=F('certificate_generation') + 1) - export_caddy_config(CADDY_EXPORT_DIR) - if settings.DEBUG: - export_caddy_config(os.path.join(settings.BASE_DIR, 'containers', 'caddy', 'config_files')) + apply_caddy_config() messages.success(request, _( 'A new certificate will be requested for %(hostname)s. The current configuration was exported.' @@ -564,6 +637,7 @@ def view_manage_dns_credential(request): if form.is_valid(): form.save() messages.success(request, _('DNS Credential saved successfully.')) + auto_apply_caddy_config(request) return redirect(cancel_url) context = { @@ -587,6 +661,7 @@ def view_delete_dns_credential(request): try: dns_credential.delete() messages.success(request, _('DNS Credential deleted successfully.')) + auto_apply_caddy_config(request) except ProtectedError: messages.error(request, _('Cannot delete this DNS Credential because it is currently in use by an Application Host.')) return redirect(cancel_url) @@ -618,10 +693,6 @@ def view_export_caddy_config(request): )) return redirect(redirect_url) - export_caddy_config(CADDY_EXPORT_DIR) - - if settings.DEBUG: - export_caddy_config(os.path.join(settings.BASE_DIR, 'containers', 'caddy', 'config_files')) - + apply_caddy_config() messages.success(request, _('Configuration exported successfully.')) return redirect(redirect_url) diff --git a/cluster/admin.py b/cluster/admin.py index 9fb403c..a866e09 100644 --- a/cluster/admin.py +++ b/cluster/admin.py @@ -5,18 +5,18 @@ from .models import ClusterSettings, Worker, WorkerStatus @admin.register(ClusterSettings) class ClusterSettingsAdmin(admin.ModelAdmin): - list_display = ('name', 'enabled', 'cluster_mode', 'restart_mode', 'config_version', 'created', 'updated') - list_filter = ('enabled', 'cluster_mode', 'restart_mode', 'primary_enable_wireguard') + list_display = ('name', 'enabled', 'cluster_mode', 'created', 'updated') + list_filter = ('enabled', 'cluster_mode', 'primary_enable_wireguard') @admin.register(Worker) class WorkerAdmin(admin.ModelAdmin): list_display = ('name', 'enabled', 'ip_address', 'country', 'city', 'ip_lock', 'created', 'updated') - list_filter = ('enabled', 'ip_lock', 'country', 'force_reload', 'force_restart') + list_filter = ('enabled', 'ip_lock', 'country') @admin.register(WorkerStatus) class WorkerStatusAdmin(admin.ModelAdmin): - list_display = ('worker', 'last_seen', 'config_version', 'last_reload', 'last_restart') - list_filter = ('last_seen', 'last_reload', 'last_restart') + list_display = ('worker', 'last_seen', 'worker_version', 'wireguard_status_updated') + list_filter = ('last_seen',) diff --git a/cluster/cluster_api.py b/cluster/cluster_api.py index 0b5c5f7..7438bdb 100644 --- a/cluster/cluster_api.py +++ b/cluster/cluster_api.py @@ -1,220 +1,170 @@ -import glob import json -import os from django.conf import settings -from django.http import JsonResponse, FileResponse +from django.core.exceptions import ValidationError +from django.http import JsonResponse from django.utils import timezone from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.gzip import gzip_page +from django.views.decorators.http import require_GET, require_POST -from .models import ClusterSettings, Worker, WorkerStatus +from cluster.models import ClusterSettings, Worker, WorkerStatus +from cluster.sync import SECTION_NAMES, build_manifest, build_section + +REPORT_STATUS_FIELDS = ('status', 'message') def get_ip_address(request): ip_address = request.META.get('HTTP_X_FORWARDED_FOR') if ip_address: - ip_address = ip_address.split(',')[0] + ip_address = ip_address.split(',')[0].strip() else: ip_address = request.META.get('REMOTE_ADDR') return ip_address -def get_cluster_settings(): - cluster_settings, created = ClusterSettings.objects.get_or_create(name='cluster_settings') - return { - 'enabled': cluster_settings.enabled, - 'primary_enable_wireguard': cluster_settings.primary_enable_wireguard, - 'stats_sync_interval': settings.WIREGUARD_STATUS_CACHE_REFRESH_INTERVAL, - 'cluster_mode': cluster_settings.cluster_mode, - 'restart_mode': cluster_settings.restart_mode, - 'config_version': cluster_settings.config_version, - 'dns_version': cluster_settings.dns_version, - } +def _error_response(message, status): + return JsonResponse({'status': 'error', 'message': message}, status=status) def get_worker(request): - success = True - ip_address = get_ip_address(request) - token = request.GET.get('token', '') + """ + Authenticate the worker from the Authorization header and update its status. + Returns (worker, None) on success or (None, JsonResponse) when the request must be rejected. + """ + authorization = request.headers.get('Authorization', '') + if not authorization.startswith('Bearer '): + return None, _error_response('Worker not found', 403) try: - worker = Worker.objects.get(token=token) - except: - return None, False + worker = Worker.objects.get(token=authorization[len('Bearer '):].strip()) + except (Worker.DoesNotExist, ValidationError, ValueError): + return None, _error_response('Worker not found', 403) - worker_status, created = WorkerStatus.objects.get_or_create(worker=worker) + worker_status, worker_status_created = WorkerStatus.objects.get_or_create(worker=worker) try: - worker_config_version = int(request.GET.get('worker_config_version')) - worker_version = int(request.GET.get('worker_version')) - worker_dns_version = int(request.GET.get('worker_dns_version')) - except: - worker.error_status = 'missing_version' - worker.save() - return worker, False - - if worker.error_status == 'missing_version': - worker.error_status = '' - worker.save() - - if worker_version < settings.CLUSTER_WORKER_MINIMUM_VERSION: - worker.error_status = 'update_required' - worker.save() - return worker, False - if worker.error_status == 'update_required': - worker.error_status = '' - worker.save() - - if worker_status.config_version != worker_config_version: - worker_status.config_version = worker_config_version - if worker_status.worker_version != worker_version: - worker_status.worker_version = worker_version - if worker_status.dns_version != worker_dns_version: - worker_status.dns_version = worker_dns_version + worker_version = int(request.headers.get('X-Worker-Version', '0')) + except ValueError: + worker_version = 0 + worker_status.worker_version = worker_version worker_status.last_seen = timezone.now() - worker_status.save() - - if not worker.ip_address: + worker_status.save(update_fields=['worker_version', 'last_seen', 'updated']) + + ip_address = get_ip_address(request) + error_status = '' + if worker_version < settings.CLUSTER_WORKER_MINIMUM_VERSION: + error_status = 'update_required' + elif worker.ip_lock and worker.ip_address and worker.ip_address != ip_address: + error_status = 'ip_lock' + elif not worker.enabled: + error_status = 'worker_disabled' + else: + cluster_settings, cluster_settings_created = ClusterSettings.objects.get_or_create(name='cluster_settings') + if cluster_settings.enabled and not settings.WIREGUARD_STATUS_CACHE_ENABLED: + cluster_settings.enabled = False + cluster_settings.save() + if not cluster_settings.enabled: + error_status = 'cluster_disabled' + + update_fields = [] + if not worker.ip_lock and worker.ip_address != ip_address: worker.ip_address = ip_address - worker.save() + update_fields.append('ip_address') + if worker.error_status != error_status: + worker.error_status = error_status + update_fields.append('error_status') + if update_fields: + worker.save(update_fields=update_fields + ['updated']) - if worker.ip_lock: - if worker.ip_address == ip_address: - if worker.error_status == 'ip_lock': - worker.error_status = '' - worker.save() - else: - worker.error_status = 'ip_lock' - worker.save() - success = False - else: - if worker.ip_address != ip_address: - worker.ip_address = ip_address - worker.save() - - if worker.enabled: - if worker.error_status == 'worker_disabled': - worker.error_status = '' - worker.save() - else: - worker.error_status = 'worker_disabled' - worker.save() - success = False - - cluster_settings, created = ClusterSettings.objects.get_or_create(name='cluster_settings') - if not settings.WIREGUARD_STATUS_CACHE_ENABLED and cluster_settings.enabled: - cluster_settings.enabled = False - cluster_settings.save() - - if cluster_settings.enabled: - if worker.error_status == 'cluster_disabled': - worker.error_status = '' - worker.save() - else: - worker.error_status = 'cluster_disabled' - worker.save() - success = False - - return worker, success + if error_status: + return None, _error_response(error_status, 400) + return worker, None -def api_get_worker_dnsmasq_config(request): - dnsmasq_file = "/etc/dnsmasq/dnsmasq_config.tar.gz" - worker, success = get_worker(request) - if worker: - if worker.error_status or not success: - data = {'status': 'error', 'message': worker.error_status} - return JsonResponse(data, status=400) - else: - data = {'status': 'error', 'message': 'Worker not found'} - return JsonResponse(data, status=403) +def api_legacy_cluster_status(request): + # Workers before version 12 poll this endpoint. A 403 makes them shut down WireGuard and stop polling. + return JsonResponse({ + 'status': 'error', + 'message': 'update_required', + 'minimum_worker_version': settings.CLUSTER_WORKER_MINIMUM_VERSION, + }, status=403) - if not os.path.exists(dnsmasq_file): - data = {'status': 'error', 'message': 'dnsmasq configuration not found'} - return JsonResponse(data, status=404) - response = FileResponse(open(dnsmasq_file, "rb"), content_type="application/gzip") - response["Content-Disposition"] = 'attachment; filename="dnsmasq_config.tar.gz"' - response["Content-Length"] = str(os.path.getsize(dnsmasq_file)) - return response +@require_GET +def api_manifest(request): + worker, error_response = get_worker(request) + if error_response: + return error_response + return JsonResponse({ + 'status': 'success', + 'worker_minimum_version': settings.CLUSTER_WORKER_MINIMUM_VERSION, + 'sections': build_manifest(worker), + }) + + +@gzip_page +@require_GET +def api_section(request, section_name): + worker, error_response = get_worker(request) + if error_response: + return error_response + if section_name not in SECTION_NAMES: + return _error_response('Section not found', 404) + return JsonResponse({'status': 'success', 'section': section_name, **build_section(section_name, worker).as_dict()}) + + +def _clean_section_report(sections): + cleaned_sections = {} + for section_name, report in sections.items(): + if section_name not in SECTION_NAMES or not isinstance(report, dict): + continue + status = report.get('status') + if status not in ('ok', 'error'): + continue + cleaned_sections[section_name] = { + 'hash': str(report.get('hash', ''))[:64], + 'status': status, + 'message': str(report.get('message', ''))[:1000], + 'updated': timezone.now().isoformat(), + } + return cleaned_sections + @csrf_exempt -def api_submit_worker_wireguard_stats(request): - worker, success = get_worker(request) - if worker: - if worker.error_status or not success: - data = {'status': 'error', 'message': worker.error_status} - return JsonResponse(data, status=400) - else: - data = {'status': 'error', 'message': 'Worker not found'} - return JsonResponse(data, status=403) - +@require_POST +def api_report(request): + worker, error_response = get_worker(request) + if error_response: + return error_response try: - if request.method == 'POST': - payload = json.loads(request.body) - worker_status = worker.workerstatus - worker_status.wireguard_status = payload - worker_status.wireguard_status_updated = timezone.now() - worker_status.save() - data = {'status': 'success', 'message': 'Stats received'} - return JsonResponse(data, status=200) - except Exception as e: - pass + payload = json.loads(request.body) + except (ValueError, UnicodeDecodeError): + return _error_response('Invalid JSON', 400) + if not isinstance(payload, dict): + return _error_response('Invalid JSON', 400) - data = {'status': 'error', 'message': 'Stats not received'} - return JsonResponse(data, status=400) + worker_status = worker.workerstatus + update_fields = [] + if isinstance(payload.get('sections'), dict): + section_status = dict(worker_status.section_status or {}) + section_status.update(_clean_section_report(payload['sections'])) + worker_status.section_status = section_status + update_fields.append('section_status') -def api_get_worker_config_files(request): - worker, success = get_worker(request) - if worker: - if worker.error_status or not success: - data = {'status': 'error', 'message': worker.error_status} - return JsonResponse(data, status=400) - else: - data = {'status': 'error', 'message': 'Worker not found'} - return JsonResponse(data, status=403) + if isinstance(payload.get('services'), dict): + worker_status.services_status = payload['services'] + update_fields.append('services_status') - config_files = ( - glob.glob('/etc/wireguard/wg*.conf') + - glob.glob('/etc/wireguard/wg-firewall.sh') - ) + if isinstance(payload.get('certificates'), dict): + worker_status.certificate_status = payload['certificates'] + worker_status.certificate_status_updated = timezone.now() + update_fields += ['certificate_status', 'certificate_status_updated'] - files = {} + if isinstance(payload.get('wireguard_status'), dict): + worker_status.wireguard_status = payload['wireguard_status'] + worker_status.wireguard_status_updated = timezone.now() + update_fields += ['wireguard_status', 'wireguard_status_updated'] - for path in config_files: - filename = os.path.basename(path) - with open(path, 'r') as f: - files[filename] = f.read() - - data = {'status': 'success', 'files': files, 'cluster_settings': get_cluster_settings()} - return JsonResponse(data, status=200) - - -def api_worker_ping(request): - worker, success = get_worker(request) - if worker: - if worker.error_status or not success: - data = {'status': 'error', 'message': worker.error_status} - return JsonResponse(data, status=400) - else: - data = {'status': 'error', 'message': 'Worker not found'} - return JsonResponse(data, status=403) - - data = { - 'status': 'success', - 'worker_error_status': worker.error_status, - } - - return JsonResponse(data, status=200) - - -def api_cluster_status(request): - worker, success = get_worker(request) - if worker: - if worker.error_status or not success: - data = {'status': 'error', 'message': worker.error_status} - return JsonResponse(data, status=400) - else: - data = {'status': 'error', 'message': 'Worker not found'} - return JsonResponse(data, status=403) - data = {'status': 'success', 'worker_error_status': worker.error_status, 'cluster_settings': get_cluster_settings()} - return JsonResponse(data, status=200) + if update_fields: + worker_status.save(update_fields=update_fields + ['updated']) + return JsonResponse({'status': 'success'}) diff --git a/cluster/forms.py b/cluster/forms.py index b3c2799..0c9e546 100644 --- a/cluster/forms.py +++ b/cluster/forms.py @@ -31,8 +31,10 @@ class WorkerForm(forms.ModelForm): if self.instance.pk: delete_html = f"{delete_label}" + back_url = f"/cluster/worker/view/?uuid={self.instance.uuid}" else: delete_html = '' + back_url = '/cluster/' self.helper.layout = Layout( Row( @@ -65,7 +67,7 @@ class WorkerForm(forms.ModelForm): Row( Column( Submit('submit', _('Save'), css_class='btn btn-success'), - HTML(f' {back_label} '), + HTML(f' {back_label} '), HTML(delete_html), css_class='col-md-12'), css_class='form-row' @@ -90,14 +92,13 @@ class WorkerForm(forms.ModelForm): class ClusterSettingsForm(forms.ModelForm): class Meta: model = ClusterSettings - fields = ['enabled', 'primary_enable_wireguard', 'cluster_mode', 'restart_mode', 'worker_display'] + fields = ['enabled', 'primary_enable_wireguard', 'cluster_mode', 'worker_display'] def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.fields['enabled'].label = _("Cluster Enabled") self.fields['primary_enable_wireguard'].label = _("Primary Enable WireGuard") self.fields['cluster_mode'].label = _("Cluster Mode") - self.fields['restart_mode'].label = _("Restart Mode") self.fields['worker_display'].label = _("Worker Display") back_label = _("Back") @@ -112,11 +113,7 @@ class ClusterSettingsForm(forms.ModelForm): ), Row( Column('cluster_mode', css_class='form-group col-md-6 mb-0'), - Column('restart_mode', css_class='form-group col-md-6 mb-0'), - css_class='form-row' - ), - Row( - Column('worker_display', css_class='form-group col-md-12 mb-0'), + Column('worker_display', css_class='form-group col-md-6 mb-0'), css_class='form-row' ), Row( diff --git a/cluster/migrations/0018_cluster_v2.py b/cluster/migrations/0018_cluster_v2.py new file mode 100644 index 0000000..d195940 --- /dev/null +++ b/cluster/migrations/0018_cluster_v2.py @@ -0,0 +1,78 @@ +# Generated by Django 5.2.19 on 2026-11-19 16:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('cluster', '0017_alter_workerstatus_wireguard_status'), + ] + + operations = [ + migrations.RemoveField( + model_name='clustersettings', + name='config_version', + ), + migrations.RemoveField( + model_name='clustersettings', + name='dns_version', + ), + migrations.RemoveField( + model_name='clustersettings', + name='restart_mode', + ), + migrations.RemoveField( + model_name='worker', + name='force_reload', + ), + migrations.RemoveField( + model_name='worker', + name='force_restart', + ), + migrations.RemoveField( + model_name='workerstatus', + name='active_peers', + ), + migrations.RemoveField( + model_name='workerstatus', + name='config_version', + ), + migrations.RemoveField( + model_name='workerstatus', + name='dns_version', + ), + migrations.RemoveField( + model_name='workerstatus', + name='last_reload', + ), + migrations.RemoveField( + model_name='workerstatus', + name='last_restart', + ), + migrations.AddField( + model_name='workerstatus', + name='certificate_status', + field=models.JSONField(blank=True, default=dict), + ), + migrations.AddField( + model_name='workerstatus', + name='certificate_status_updated', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name='workerstatus', + name='section_status', + field=models.JSONField(blank=True, default=dict), + ), + migrations.AddField( + model_name='workerstatus', + name='services_status', + field=models.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='worker', + name='error_status', + field=models.CharField(choices=[('', ''), ('ip_lock', 'IP lock is enabled, but the worker is attempting to access from a different IP address.'), ('worker_disabled', 'Worker is not enabled'), ('cluster_disabled', 'Cluster is not enabled'), ('update_required', 'Worker update is required.')], default='', max_length=32), + ), + ] diff --git a/cluster/models.py b/cluster/models.py index 591904d..81069d0 100644 --- a/cluster/models.py +++ b/cluster/models.py @@ -9,15 +9,12 @@ class ClusterSettings(models.Model): enabled = models.BooleanField(default=False) primary_enable_wireguard = models.BooleanField(default=True) cluster_mode = models.CharField(default='mirror', max_length=16, choices=(('mirror', 'Mirror'), )) - restart_mode = models.CharField(default='auto', max_length=16, choices=(('auto', 'Automatic restart'),)) worker_display = models.CharField( default='server_address', max_length=16, choices=( ('name', 'Name'), ('server_address', 'Server Address'), ('location', 'Location'), ('address_location', 'Address + Location') ) ) - config_version = models.PositiveIntegerField(default=0) - dns_version = models.PositiveIntegerField(default=0) updated = models.DateTimeField(auto_now=True) created = models.DateTimeField(auto_now_add=True) @@ -26,6 +23,10 @@ class ClusterSettings(models.Model): def __str__(self): return self.name + @classmethod + def cluster_enabled(cls): + return cls.objects.filter(name='cluster_settings', enabled=True).exists() + class Worker(models.Model): name = models.CharField(max_length=100, unique=True) @@ -34,9 +35,6 @@ class Worker(models.Model): ip_lock = models.BooleanField(default=False) ip_address = models.GenericIPAddressField(blank=True, null=True) - force_reload = models.BooleanField(default=False) - force_restart = models.BooleanField(default=False) - country = models.CharField(max_length=100, blank=True, null=True) city = models.CharField(max_length=100, blank=True, null=True) hostname = models.CharField(max_length=100, blank=True, null=True) @@ -46,7 +44,6 @@ class Worker(models.Model): ('ip_lock', _('IP lock is enabled, but the worker is attempting to access from a different IP address.')), ('worker_disabled', _('Worker is not enabled')), ('cluster_disabled', _('Cluster is not enabled')), - ('missing_version', _('Please report worker_config_version, worker_dns_version and worker_version in the API request.')), ('update_required', _('Worker update is required.')) )) @@ -105,14 +102,15 @@ class Worker(models.Model): class WorkerStatus(models.Model): worker = models.OneToOneField(Worker, on_delete=models.CASCADE) last_seen = models.DateTimeField(auto_now=True) - last_reload = models.DateTimeField(blank=True, null=True) - last_restart = models.DateTimeField(blank=True, null=True) - dns_version = models.PositiveIntegerField(default=0) - config_version = models.PositiveIntegerField(default=0) worker_version = models.PositiveIntegerField(default=0) - active_peers = models.PositiveIntegerField(default=0) wireguard_status = models.JSONField(blank=True, null=True) wireguard_status_updated = models.DateTimeField(blank=True, null=True) + # {section: {"hash": ..., "status": "ok"|"error", "message": ..., "updated": ...}} + section_status = models.JSONField(default=dict, blank=True) + services_status = models.JSONField(default=dict, blank=True) + # {hostname: {"status": ..., "issuer": ..., "not_after": ..., "generation": ...}} + certificate_status = models.JSONField(default=dict, blank=True) + certificate_status_updated = models.DateTimeField(blank=True, null=True) updated = models.DateTimeField(auto_now=True) created = models.DateTimeField(auto_now_add=True) diff --git a/cluster/sync.py b/cluster/sync.py new file mode 100644 index 0000000..6ddf0f8 --- /dev/null +++ b/cluster/sync.py @@ -0,0 +1,209 @@ +import glob +import hashlib +import json +import os +from datetime import timedelta + +from django.conf import settings +from django.utils import timezone +from django.utils.dateparse import parse_datetime + +from app_gateway.caddy_config_export import ( + add_certificate_files, certificate_file_name, load_worker_export, read_exported_certificate +) +from cluster.models import WorkerStatus +from dns.functions import generate_dnsmasq_config +from dns.models import DNSFilterList +from firewall.tools import generate_firewall_script + +WIREGUARD_CONFIG_DIR = '/etc/wireguard' +DNSMASQ_CONFIG_DIR = '/etc/dnsmasq' +DNSMASQ_MAIN_CONFIG = 'wireguard_webadmin_dns.conf' + +# Sections are applied by the node in this order. The firewall comes first because the WireGuard +# configuration runs wg-firewall.sh as PostUp of one of the instances, and certificates come before the +# gateway so the generated Caddyfile only references files that already exist. +SECTION_NAMES = ('firewall', 'wireguard', 'dns', 'settings', 'certificates', 'gateway') + +# Large files (e.g. DNS filter lists) are only read and hashed again when they change on disk +_file_cache = {} + + +def _sha256(text): + return hashlib.sha256(text.encode()).hexdigest() + + +def _read_file(path): + file_stat = os.stat(path) + cache_key = (file_stat.st_mtime_ns, file_stat.st_size) + cached = _file_cache.get(path) + if cached and cached['key'] == cache_key: + return cached['content'], cached['sha256'] + with open(path, 'r', encoding='utf-8', errors='replace') as config_file: + content = config_file.read() + content_sha256 = _sha256(content) + _file_cache[path] = {'key': cache_key, 'content': content, 'sha256': content_sha256} + return content, content_sha256 + + +class Section: + def __init__(self): + self.files = {} + self.file_hashes = {} + self.values = {} + + def add_file(self, filename, content, content_sha256=None): + self.files[filename] = content + self.file_hashes[filename] = content_sha256 or _sha256(content) + + @property + def hash(self): + fingerprint = json.dumps({'files': self.file_hashes, 'values': self.values}, sort_keys=True) + return _sha256(fingerprint) + + def as_dict(self): + return {'hash': self.hash, 'files': self.files, 'values': self.values} + + +def build_wireguard_section(worker): + section = Section() + for config_path in sorted(glob.glob(os.path.join(WIREGUARD_CONFIG_DIR, 'wg[0-9]*.conf'))): + content, content_sha256 = _read_file(config_path) + section.add_file(os.path.basename(config_path), content, content_sha256) + return section + + +def build_firewall_section(worker): + section = Section() + section.add_file('wg-firewall.sh', generate_firewall_script(cluster_node=True)) + return section + + +def build_dns_section(worker): + section = Section() + for dns_list in DNSFilterList.objects.filter(enabled=True).order_by('uuid'): + list_path = os.path.join(DNSMASQ_CONFIG_DIR, f'{dns_list.uuid}.conf') + if os.path.isfile(list_path): + content, content_sha256 = _read_file(list_path) + section.add_file(os.path.basename(list_path), content, content_sha256) + section.add_file(DNSMASQ_MAIN_CONFIG, generate_dnsmasq_config(cluster_node=True)) + return section + + +def build_settings_section(worker): + section = Section() + section.values = { + 'stats_sync_interval': settings.WIREGUARD_STATUS_CACHE_REFRESH_INTERVAL, + } + return section + + +EMPTY_GATEWAY_EXPORT = { + 'applications.json': {'entries': []}, + 'auth_policies.json': {'auth_methods': {}, 'groups': {}, 'users': {}, 'policies': {}}, + 'routes.json': {'entries': {}}, + 'tls.json': {'hosts': {}, 'dns_credentials': {}}, +} + + +def get_gateway_export(worker): + """Configuration exported for the worker, with the certificate files currently available on the primary.""" + gateway_export = load_worker_export(worker) or json.loads(json.dumps(EMPTY_GATEWAY_EXPORT)) + add_certificate_files(gateway_export['tls.json']) + return gateway_export + + +def build_certificates_section(worker): + """Certificate and private key of every ACME host served by the worker, named after the certificate hash.""" + section = Section() + for hostname, host_tls in sorted(get_gateway_export(worker)['tls.json']['hosts'].items()): + if not host_tls.get('certificate_file'): + continue + certificate, private_key = read_exported_certificate(hostname) + if not certificate or not private_key: + continue + file_name = certificate_file_name(certificate) + section.add_file(f'{file_name}.crt', certificate) + section.add_file(f'{file_name}.key', private_key) + return section + + +def build_gateway_section(worker): + section = Section() + for file_name, data in get_gateway_export(worker).items(): + section.add_file(file_name, json.dumps(data, indent=2) + '\n') + return section + + +SECTION_BUILDERS = { + 'firewall': build_firewall_section, + 'wireguard': build_wireguard_section, + 'dns': build_dns_section, + 'settings': build_settings_section, + 'certificates': build_certificates_section, + 'gateway': build_gateway_section, +} + + +def build_section(section_name, worker): + return SECTION_BUILDERS[section_name](worker) + + +def build_manifest(worker): + return {section_name: build_section(section_name, worker).hash for section_name in SECTION_NAMES} + + +CERTIFICATE_EXPIRATION_WARNING_DAYS = 14 + + +def get_worker_warnings(worker): + """Stopped services and certificates that are not trusted or expire soon, as reported by the worker.""" + try: + worker_status = worker.workerstatus + except WorkerStatus.DoesNotExist: + return {'stopped_services': [], 'certificate_issues': []} + + stopped_services = sorted( + service for service, state in (worker_status.services_status or {}).items() if state != 'running' + ) + expiration_limit = timezone.now() + timedelta(days=CERTIFICATE_EXPIRATION_WARNING_DAYS) + certificate_issues = [] + for hostname, certificate in sorted((worker_status.certificate_status or {}).items()): + not_after = parse_datetime(certificate.get('not_after') or '') + if certificate.get('status') not in ('trusted', 'untrusted'): + certificate_issues.append(hostname) + elif certificate.get('status') == 'trusted' and not_after and not_after < expiration_limit: + certificate_issues.append(hostname) + return {'stopped_services': stopped_services, 'certificate_issues': certificate_issues} + + +def get_worker_sync_state(worker, manifest=None): + """ + Compare the hashes reported by the worker with the current ones. + Returns {'state': 'synced'|'pending'|'error', 'sections': {name: {'state': ..., 'message': ...}}}. + """ + manifest = manifest or build_manifest(worker) + try: + section_status = worker.workerstatus.section_status or {} + except WorkerStatus.DoesNotExist: + section_status = {} + + sections = {} + for section_name, section_hash in manifest.items(): + reported = section_status.get(section_name) or {} + if reported.get('hash') != section_hash: + # A previous error is kept as information while the node retries with the new content + sections[section_name] = {'state': 'pending', 'message': reported.get('message', '')} + elif reported.get('status') == 'error': + sections[section_name] = {'state': 'error', 'message': reported.get('message', '')} + else: + sections[section_name] = {'state': 'synced', 'message': ''} + + states = {section['state'] for section in sections.values()} + if 'error' in states: + state = 'error' + elif 'pending' in states: + state = 'pending' + else: + state = 'synced' + return {'state': state, 'sections': sections} diff --git a/cluster/tests.py b/cluster/tests.py index a39b155..4937496 100644 --- a/cluster/tests.py +++ b/cluster/tests.py @@ -1 +1,273 @@ -# Create your tests here. +import json +import os +import tempfile +from unittest.mock import patch + +from django.test import TestCase, override_settings +from django.utils import timezone + +from api.models import WireguardStatusCache +from api.views import func_concatenate_cluster_wireguard_status_cache +from app_gateway import caddy_config_export +from app_gateway.caddy_config_export import certificate_file_name, export_caddy_config +from app_gateway.models import Application, ApplicationHost, DnsChallengeCredential +from cluster import sync +from cluster.models import ClusterSettings, Worker, WorkerStatus +from dns.models import DNSSettings +from firewall.tools import generate_firewall_script + +FAKE_CERTIFICATE = '-----BEGIN CERTIFICATE-----\nfake\n-----END CERTIFICATE-----\n' +FAKE_PRIVATE_KEY = '-----BEGIN PRIVATE KEY-----\nfake\n-----END PRIVATE KEY-----\n' + + +class ClusterTestMixin: + """Isolated directories for the files read and written by the cluster sync and the Caddy export.""" + + def setUp(self): + super().setUp() + self.temp_dir = tempfile.TemporaryDirectory() + self.addCleanup(self.temp_dir.cleanup) + self.wireguard_dir = os.path.join(self.temp_dir.name, 'wireguard') + self.dnsmasq_dir = os.path.join(self.temp_dir.name, 'dnsmasq') + self.export_dir = os.path.join(self.temp_dir.name, 'caddy_json_export') + self.certificate_export_dir = os.path.join(self.temp_dir.name, 'caddy_cert_export') + for directory in (self.wireguard_dir, self.dnsmasq_dir, self.export_dir, self.certificate_export_dir): + os.makedirs(directory) + with open(os.path.join(self.wireguard_dir, 'wg0.conf'), 'w', encoding='utf-8') as config_file: + config_file.write('[Interface]\nListenPort = 51820\n') + + for target, value in ( + ('cluster.sync.WIREGUARD_CONFIG_DIR', self.wireguard_dir), + ('cluster.sync.DNSMASQ_CONFIG_DIR', self.dnsmasq_dir), + ('app_gateway.caddy_config_export.CADDY_EXPORT_DIR', self.export_dir), + ('app_gateway.caddy_config_export.CERTIFICATE_EXPORT_DIR', self.certificate_export_dir), + ): + patcher = patch(target, value) + patcher.start() + self.addCleanup(patcher.stop) + + DNSSettings.objects.get_or_create(name='dns_settings') + ClusterSettings.objects.update_or_create(name='cluster_settings', defaults={'enabled': True}) + self.worker = Worker.objects.create(name='node-a', hostname='node-a.example.test') + self.headers = {'HTTP_AUTHORIZATION': f'Bearer {self.worker.token}', 'HTTP_X_WORKER_VERSION': '12'} + + def write_exported_certificate(self, hostname): + host_dir = os.path.join(self.certificate_export_dir, hostname) + os.makedirs(host_dir, exist_ok=True) + with open(os.path.join(host_dir, 'certificate.crt'), 'w', encoding='utf-8') as certificate_file: + certificate_file.write(FAKE_CERTIFICATE) + with open(os.path.join(host_dir, 'certificate.key'), 'w', encoding='utf-8') as key_file: + key_file.write(FAKE_PRIVATE_KEY) + + +@override_settings(CLUSTER_WORKER_MINIMUM_VERSION=12, WIREGUARD_STATUS_CACHE_ENABLED=True) +class ClusterApiTests(ClusterTestMixin, TestCase): + def test_legacy_status_endpoint_stops_old_workers_without_token(self): + response = self.client.get('/api/cluster/status/') + self.assertEqual(response.status_code, 403) + self.assertEqual(response.json()['message'], 'update_required') + self.assertEqual(response.json()['minimum_worker_version'], 12) + + def test_manifest_requires_token(self): + self.assertEqual(self.client.get('/api/cluster/v2/manifest/').status_code, 403) + response = self.client.get('/api/cluster/v2/manifest/', HTTP_AUTHORIZATION='Bearer not-a-uuid') + self.assertEqual(response.status_code, 403) + + def test_worker_is_refused_with_reason(self): + scenarios = ( + ({'HTTP_X_WORKER_VERSION': '11'}, {}, 'update_required'), + ({}, {'enabled': False}, 'worker_disabled'), + ({}, {'ip_lock': True, 'ip_address': '203.0.113.10'}, 'ip_lock'), + ) + for header_override, worker_fields, expected_error in scenarios: + with self.subTest(expected_error=expected_error): + Worker.objects.filter(pk=self.worker.pk).update(**{'enabled': True, 'ip_lock': False, **worker_fields}) + response = self.client.get('/api/cluster/v2/manifest/', **{**self.headers, **header_override}) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json()['message'], expected_error) + self.worker.refresh_from_db() + self.assertEqual(self.worker.error_status, expected_error) + + def test_disabled_cluster_refuses_workers(self): + ClusterSettings.objects.filter(name='cluster_settings').update(enabled=False) + response = self.client.get('/api/cluster/v2/manifest/', **self.headers) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json()['message'], 'cluster_disabled') + + def test_manifest_hashes_match_sections(self): + response = self.client.get('/api/cluster/v2/manifest/', **self.headers) + self.assertEqual(response.status_code, 200) + manifest = response.json()['sections'] + self.assertEqual(list(manifest), list(sync.SECTION_NAMES)) + for section_name, section_hash in manifest.items(): + with self.subTest(section=section_name): + section_response = self.client.get(f'/api/cluster/v2/section/{section_name}/', **self.headers) + self.assertEqual(section_response.status_code, 200) + self.assertEqual(section_response.json()['hash'], section_hash) + self.assertEqual(self.client.get('/api/cluster/v2/section/unknown/', **self.headers).status_code, 404) + + def test_report_stores_worker_status(self): + report = { + 'sections': { + 'dns': {'hash': 'abc', 'status': 'error', 'message': 'dnsmasq failed'}, + 'unknown': {'hash': 'abc', 'status': 'ok'}, + 'settings': {'hash': 'def', 'status': 'invalid'}, + }, + 'services': {'dnsmasq': 'running'}, + 'certificates': {'app.example.test': {'status': 'trusted'}}, + 'wireguard_status': {'wg0': {}}, + } + response = self.client.post( + '/api/cluster/v2/report/', data=json.dumps(report), content_type='application/json', **self.headers + ) + self.assertEqual(response.status_code, 200) + worker_status = WorkerStatus.objects.get(worker=self.worker) + self.assertEqual(list(worker_status.section_status), ['dns']) + self.assertEqual(worker_status.section_status['dns']['message'], 'dnsmasq failed') + self.assertEqual(worker_status.services_status, {'dnsmasq': 'running'}) + self.assertEqual(worker_status.certificate_status, report['certificates']) + self.assertEqual(worker_status.wireguard_status, {'wg0': {}}) + self.assertIsNotNone(worker_status.certificate_status_updated) + self.assertIsNotNone(worker_status.wireguard_status_updated) + + +class ClusterSyncTests(ClusterTestMixin, TestCase): + def report_sections(self, manifest, overrides=None): + section_status = {name: {'hash': section_hash, 'status': 'ok'} for name, section_hash in manifest.items()} + section_status.update(overrides or {}) + WorkerStatus.objects.update_or_create(worker=self.worker, defaults={'section_status': section_status}) + self.worker.refresh_from_db() + + def test_sync_state(self): + manifest = sync.build_manifest(self.worker) + self.assertEqual(sync.get_worker_sync_state(self.worker, manifest)['state'], 'pending') + + self.report_sections(manifest) + self.assertEqual(sync.get_worker_sync_state(self.worker, manifest)['state'], 'synced') + + self.report_sections(manifest, {'dns': {'hash': manifest['dns'], 'status': 'error', 'message': 'failed'}}) + sync_state = sync.get_worker_sync_state(self.worker, manifest) + self.assertEqual(sync_state['state'], 'error') + self.assertEqual(sync_state['sections']['dns']['message'], 'failed') + + # An error reported for older content means the node still has to try the new one + self.report_sections(manifest, {'dns': {'hash': 'old', 'status': 'error', 'message': 'failed'}}) + self.assertEqual(sync.get_worker_sync_state(self.worker, manifest)['state'], 'pending') + + def test_wireguard_section_follows_config_files(self): + first_hash = sync.build_section('wireguard', self.worker).hash + self.assertEqual(sync.build_section('wireguard', self.worker).hash, first_hash) + with open(os.path.join(self.wireguard_dir, 'wg1.conf'), 'w', encoding='utf-8') as config_file: + config_file.write('[Interface]\nListenPort = 51821\n') + section = sync.build_section('wireguard', self.worker) + self.assertNotEqual(section.hash, first_hash) + self.assertEqual(sorted(section.files), ['wg0.conf', 'wg1.conf']) + + def test_dns_section_uses_node_configuration(self): + section = sync.build_section('dns', self.worker) + main_config = section.files[sync.DNSMASQ_MAIN_CONFIG] + self.assertIn('bind-dynamic', main_config) + self.assertIn('interface=wg*', main_config) + self.assertNotIn('listen-address=0.0.0.0', main_config) + + def test_firewall_section_is_stable_and_node_specific(self): + self.assertEqual(sync.build_section('firewall', self.worker).hash, sync.build_section('firewall', self.worker).hash) + node_script = generate_firewall_script(cluster_node=True) + primary_script = generate_firewall_script() + self.assertNotIn('DNS_IP', node_script) + self.assertNotIn('--dport 8000', node_script) + self.assertNotIn('generated by', node_script) + self.assertIn('DNS_IP', primary_script) + + +class ClusterGatewaySyncTests(ClusterTestMixin, TestCase): + def setUp(self): + super().setUp() + self.credential = DnsChallengeCredential.objects.create( + display_name='Cloudflare', provider='cloudflare', credentials={'api_token': 'x' * 40} + ) + self.node_app = Application.objects.create( + display_name='Node App', upstream='http://10.0.0.20:8080', serve_on_primary=False + ) + self.node_app.workers.add(self.worker) + ApplicationHost.objects.create( + application=self.node_app, hostname='node.example.test', certificate_type='dns_challenge', + dns_credential=self.credential, + ) + + def gateway_files(self): + return { + filename: json.loads(content) + for filename, content in sync.build_section('gateway', self.worker).files.items() + } + + def test_primary_manages_certificates_of_worker_only_applications(self): + export_caddy_config(self.export_dir) + with open(os.path.join(self.export_dir, 'applications.json'), encoding='utf-8') as applications_file: + primary_applications = json.load(applications_file) + with open(os.path.join(self.export_dir, 'tls.json'), encoding='utf-8') as tls_file: + primary_tls = json.load(tls_file) + self.assertNotIn('node-app', [entry['id'] for entry in primary_applications['entries']]) + host_tls = primary_tls['hosts']['node.example.test'] + self.assertTrue(host_tls['certificate_only']) + self.assertTrue(host_tls['export_certificate']) + self.assertEqual(host_tls['dns_credential'], self.credential.name) + + def test_workers_only_publish_exported_configuration(self): + self.assertEqual(self.gateway_files()['applications.json'], {'entries': []}) + + export_caddy_config(self.export_dir) + self.assertEqual([entry['id'] for entry in self.gateway_files()['applications.json']['entries']], ['node-app']) + + self.node_app.upstream = 'http://10.0.0.21:8080' + self.node_app.save() + self.assertTrue(caddy_config_export.has_pending_export(self.export_dir)) + self.assertEqual(self.gateway_files()['applications.json']['entries'][0]['upstream'], 'http://10.0.0.20:8080') + + export_caddy_config(self.export_dir) + self.assertFalse(caddy_config_export.has_pending_export(self.export_dir)) + self.assertEqual(self.gateway_files()['applications.json']['entries'][0]['upstream'], 'http://10.0.0.21:8080') + + def test_worker_receives_exported_certificate_without_dns_credentials(self): + export_caddy_config(self.export_dir) + worker_tls = self.gateway_files()['tls.json'] + self.assertEqual(worker_tls['dns_credentials'], {}) + self.assertEqual(worker_tls['hosts']['node.example.test']['certificate_file'], '') + self.assertEqual(sync.build_section('certificates', self.worker).files, {}) + + # A renewed certificate reaches the worker without a new export + self.write_exported_certificate('node.example.test') + certificate_name = certificate_file_name(FAKE_CERTIFICATE) + self.assertEqual(self.gateway_files()['tls.json']['hosts']['node.example.test']['certificate_file'], certificate_name) + self.assertEqual(sync.build_section('certificates', self.worker).files, { + f'{certificate_name}.crt': FAKE_CERTIFICATE, + f'{certificate_name}.key': FAKE_PRIVATE_KEY, + }) + + def test_new_worker_is_pending_until_exported(self): + export_caddy_config(self.export_dir) + self.assertFalse(caddy_config_export.has_pending_export(self.export_dir)) + Worker.objects.create(name='node-b') + self.assertTrue(caddy_config_export.has_pending_export(self.export_dir)) + + +class ClusterTrafficAggregationTests(TestCase): + def test_peer_traffic_is_summed_across_servers(self): + worker = Worker.objects.create(name='node-a') + WorkerStatus.objects.create( + worker=worker, wireguard_status_updated=timezone.now(), wireguard_status={'wg0': { + 'PEER': {'transfer': {'tx': 5000, 'rx': 500}, 'latest-handshakes': '2000', 'endpoints': 'node:1'}, + 'NODE_ONLY': {'transfer': {'tx': 7, 'rx': 3}, 'latest-handshakes': '5'}, + }}, + ) + WireguardStatusCache.objects.create(cache_type='master', processing_time_ms=0, data={'wg0': { + 'PEER': {'transfer': {'tx': 100, 'rx': 10}, 'latest-handshakes': '1000', 'endpoints': 'primary:1'}, + }}) + + func_concatenate_cluster_wireguard_status_cache() + + combined = WireguardStatusCache.objects.filter(cache_type='cluster').latest('created').data['wg0'] + self.assertEqual(combined['PEER']['transfer'], {'tx': 5100, 'rx': 510}) + self.assertEqual(combined['PEER']['endpoints'], 'node:1') + self.assertEqual(combined['PEER']['source_name'], 'worker_node-a') + self.assertEqual(combined['NODE_ONLY']['transfer'], {'tx': 7, 'rx': 3}) diff --git a/cluster/urls_api.py b/cluster/urls_api.py index 2f0e1f5..f6027b4 100644 --- a/cluster/urls_api.py +++ b/cluster/urls_api.py @@ -1,12 +1,10 @@ from django.urls import path -from cluster.cluster_api import api_cluster_status, api_get_worker_config_files, api_get_worker_dnsmasq_config, \ - api_worker_ping, api_submit_worker_wireguard_stats +from cluster.cluster_api import api_legacy_cluster_status, api_manifest, api_report, api_section urlpatterns = [ - path('status/', api_cluster_status, name='api_cluster_status'), - path('worker/get_config_files/', api_get_worker_config_files, name='api_get_worker_config_files'), - path('worker/get_dnsmasq_config/', api_get_worker_dnsmasq_config, name='api_get_worker_dnsmasq_config'), - path('worker/ping/', api_worker_ping, name='api_worker_ping'), - path('worker/submit_wireguard_stats/', api_submit_worker_wireguard_stats, name='api_submit_worker_wireguard_stats'), + path('status/', api_legacy_cluster_status, name='api_legacy_cluster_status'), + path('v2/manifest/', api_manifest, name='api_cluster_manifest'), + path('v2/section/Configure a cluster worker node that will synchronize with this primary instance.
- -A unique name to identify this worker.
- -The IP address of the worker node. Leave empty if IP lock is disabled.
- -When enabled, the worker can only connect from the specified IP address.
- -Optional location details for this worker (country, city, hostname).
+A worker is a cluster node that replicates WireGuard, firewall and DNS from this primary server and, optionally, publishes App Gateway applications.
+ +Used to identify the worker. The hostname is the address clients use to connect to this worker: it is written as the endpoint of the peer configurations downloaded for this server.
+ +Secret used by the worker to authenticate with this primary server. Changing it disconnects the worker until its configuration is updated.
+ +When enabled, the worker is only accepted from the IP address below. Without it, the address is updated automatically from the last connection.
+ +Disabled workers are refused and shut down their WireGuard interfaces until enabled again.
''') } - + context = { - 'page_title': page_title, - 'form': form, - 'worker': worker, + 'page_title': page_title, + 'form': form, + 'worker': worker, 'instance': worker, - 'form_description': form_description + 'form_description': form_description, } return render(request, 'generic_form.html', context) @login_required def cluster_settings(request): - """Cluster settings configuration""" if not UserAcl.objects.filter(user=request.user).filter(user_level__gte=50).exists(): return render(request, 'access_denied.html', {'page_title': _('Access Denied')}) - - cluster_settings, created = ClusterSettings.objects.get_or_create(name='cluster_settings') - page_title = _('Cluster Settings') - - if request.method == 'POST': - form = ClusterSettingsForm(request.POST, instance=cluster_settings) - if form.is_valid(): - form.save() - messages.success(request, _('Cluster settings updated successfully.')) - if cluster_settings.enabled: - if cluster_settings.config_version == 0: - cluster_settings.config_version += 1 - cluster_settings.save() - compress_dnsmasq_config() - return redirect('/cluster/') - else: - form = ClusterSettingsForm(instance=cluster_settings) + + cluster_settings, cluster_settings_created = ClusterSettings.objects.get_or_create(name='cluster_settings') + + form = ClusterSettingsForm(request.POST or None, instance=cluster_settings) + if form.is_valid(): + form.save() + messages.success(request, _('Cluster settings updated successfully.')) + return redirect(reverse('cluster_main')) form_description = { 'size': 'col-lg-6', 'content': _(''' +When disabled, workers are refused and shut down their WireGuard interfaces, and App Gateway applications are only published by this primary server. Requires the WireGuard status cache.
+ +Lists this primary server as an option when downloading peer configurations and VPN invites.
+Configure how the cluster operates and synchronizes configurations between nodes.
- -Configure how frequently statistics and cache data are synchronized between cluster nodes.
- -Choose whether WireGuard services should be automatically restarted when configurations change, or if manual intervention is required.
- +Mirror: every worker receives the same WireGuard instances, peers, firewall and DNS configuration. Peers can connect to any server.
+Select how workers should be identified in the interface - by name, server address, location, or a combination.
+How workers are named in server selections (peer configuration downloads and VPN invites).
''') } - + context = { - 'page_title': page_title, + 'page_title': _('Cluster Settings'), 'form': form, 'cluster_settings': cluster_settings, 'instance': cluster_settings, - 'form_description': form_description + 'form_description': form_description, } return render(request, 'generic_form.html', context) diff --git a/cluster_node-dev.yml b/cluster_node-dev.yml index e13cbe8..9a973a1 100644 --- a/cluster_node-dev.yml +++ b/cluster_node-dev.yml @@ -1,4 +1,3 @@ -version: '3' services: cluster-node: container_name: cluster-node @@ -8,16 +7,19 @@ services: dockerfile: Dockerfile-cluster_node environment: - MASTER_SERVER_ADDRESS=${MASTER_SERVER_ADDRESS} - - DEBUG_MODE=${DEBUG_MODE} - - COMPOSE_VERSION=c1b + - COMPOSE_VERSION=c2a - TZ=${TIMEZONE} - TOKEN=${TOKEN} volumes: - cluster_node_wireguard:/etc/wireguard - cluster_node_dnsmasq_conf:/etc/dnsmasq/ + - cluster_node_caddy_json:/caddy_json_export/ + - cluster_node_caddy_certificates:/caddy_certificates/ ports: - # Ports for WireGuard instances. - - "51820-51839:51820-51839/udp" + # Different host ports, so the node can run on the same host as the development primary server + - "52820-52839:51820-51839/udp" + - "9080:80" + - "9443:443" cap_add: - NET_ADMIN - SYS_MODULE @@ -28,11 +30,57 @@ services: cluster-node-dns: container_name: cluster-node-dns restart: unless-stopped - image: eduardosilva/wireguard_webadmin_dns:latest + build: + context: containers/dnsmasq + dockerfile: Dockerfile-dnsmasq + network_mode: "service:cluster-node" environment: - TZ=${TIMEZONE} volumes: - cluster_node_dnsmasq_conf:/etc/dnsmasq/ + depends_on: + - cluster-node + + cluster-node-caddy: + container_name: cluster-node-caddy + restart: unless-stopped + build: + context: containers/caddy + dockerfile: Dockerfile-caddy + network_mode: "service:cluster-node" + environment: + - TZ=${TIMEZONE} + - CLUSTER_NODE=true + - AUTH_GATEWAY_INTERNAL_URL=http://cluster-node-auth-gateway:9091 + volumes: + - cluster_node_caddy_json:/caddy_json_export/ + - cluster_node_caddy_certificates:/caddy_certificates/:ro + - cluster_node_caddy_data:/data + - cluster_node_caddy_config:/config + depends_on: + - cluster-node + - cluster-node-auth-gateway + + cluster-node-auth-gateway: + container_name: cluster-node-auth-gateway + restart: unless-stopped + build: + context: containers/auth-gateway + dockerfile: Dockerfile-auth-gateway + environment: + - TZ=${TIMEZONE} + - AUTH_GATEWAY_CONFIG_DIR=/caddy_json_export + - AUTH_GATEWAY_DATABASE_PATH=/data/auth-gateway.sqlite3 + - AUTH_GATEWAY_SECURE_COOKIES=false + volumes: + - cluster_node_caddy_json:/caddy_json_export/ + - cluster_node_auth_gateway_data:/data + volumes: cluster_node_wireguard: cluster_node_dnsmasq_conf: + cluster_node_caddy_json: + cluster_node_caddy_certificates: + cluster_node_caddy_data: + cluster_node_caddy_config: + cluster_node_auth_gateway_data: diff --git a/cluster_node-gateway.yml b/cluster_node-gateway.yml new file mode 100644 index 0000000..3ecb816 --- /dev/null +++ b/cluster_node-gateway.yml @@ -0,0 +1,48 @@ +# App Gateway for cluster nodes. Use together with cluster_node.yml: +# docker compose -f cluster_node.yml -f cluster_node-gateway.yml up -d +services: + cluster-node: + volumes: + - cluster_node_caddy_json:/caddy_json_export/ + - cluster_node_caddy_certificates:/caddy_certificates/ + ports: + # Caddy shares the network of cluster-node, so its ports are published here + - "80:80" + - "443:443" + + cluster-node-caddy: + container_name: cluster-node-caddy + restart: unless-stopped + image: eduardosilva/wireguard_webadmin_caddy:latest + network_mode: "service:cluster-node" + environment: + - TZ=${TIMEZONE} + - CLUSTER_NODE=true + - AUTH_GATEWAY_INTERNAL_URL=http://cluster-node-auth-gateway:9091 + volumes: + - cluster_node_caddy_json:/caddy_json_export/ + - cluster_node_caddy_certificates:/caddy_certificates/:ro + - cluster_node_caddy_data:/data + - cluster_node_caddy_config:/config + depends_on: + - cluster-node + - cluster-node-auth-gateway + + cluster-node-auth-gateway: + container_name: cluster-node-auth-gateway + restart: unless-stopped + image: eduardosilva/wireguard_webadmin_auth_gateway:latest + environment: + - TZ=${TIMEZONE} + - AUTH_GATEWAY_CONFIG_DIR=/caddy_json_export + - AUTH_GATEWAY_DATABASE_PATH=/data/auth-gateway.sqlite3 + volumes: + - cluster_node_caddy_json:/caddy_json_export/ + - cluster_node_auth_gateway_data:/data + +volumes: + cluster_node_caddy_json: + cluster_node_caddy_certificates: + cluster_node_caddy_data: + cluster_node_caddy_config: + cluster_node_auth_gateway_data: diff --git a/cluster_node.yml b/cluster_node.yml index 5fab3de..f821bed 100644 --- a/cluster_node.yml +++ b/cluster_node.yml @@ -1,4 +1,3 @@ -version: '3' services: cluster-node: container_name: cluster-node @@ -6,8 +5,7 @@ services: image: eduardosilva/wireguard-webadmin-cluster-node:latest environment: - MASTER_SERVER_ADDRESS=${MASTER_SERVER_ADDRESS} - - DEBUG_MODE=${DEBUG_MODE} - - COMPOSE_VERSION=c1b + - COMPOSE_VERSION=c2a - TZ=${TIMEZONE} - TOKEN=${TOKEN} volumes: @@ -27,10 +25,15 @@ services: container_name: cluster-node-dns restart: unless-stopped image: eduardosilva/wireguard_webadmin_dns:latest + # Shares the network of cluster-node, so DNS is answered directly on the WireGuard interfaces + network_mode: "service:cluster-node" environment: - TZ=${TIMEZONE} volumes: - cluster_node_dnsmasq_conf:/etc/dnsmasq/ + depends_on: + - cluster-node + volumes: cluster_node_wireguard: cluster_node_dnsmasq_conf: diff --git a/containers/auth-gateway/tests/fixtures/config_files/tls.json b/containers/auth-gateway/tests/fixtures/config_files/tls.json new file mode 100644 index 0000000..dd1eb83 --- /dev/null +++ b/containers/auth-gateway/tests/fixtures/config_files/tls.json @@ -0,0 +1,40 @@ +{ + "hosts": { + "dashboard.example.test": { + "certificate_type": "http_challenge", + "certificate_fallback": true, + "certificate_generation": 0 + }, + "hypervisor.example.test": { + "certificate_type": "http_challenge", + "certificate_fallback": true, + "certificate_generation": 0 + }, + "media.example.test": { + "certificate_type": "http_challenge", + "certificate_fallback": true, + "certificate_generation": 0 + }, + "media2.example.test": { + "certificate_type": "http_challenge", + "certificate_fallback": true, + "certificate_generation": 0 + }, + "sso.example.test": { + "certificate_type": "http_challenge", + "certificate_fallback": true, + "certificate_generation": 0 + }, + "webadmin.example.test": { + "certificate_type": "http_challenge", + "certificate_fallback": true, + "certificate_generation": 0 + }, + "webadmin2.example.test": { + "certificate_type": "http_challenge", + "certificate_fallback": true, + "certificate_generation": 0 + } + }, + "dns_credentials": {} +} diff --git a/containers/caddy/entrypoint.sh b/containers/caddy/entrypoint.sh index 9aaeed9..973be03 100644 --- a/containers/caddy/entrypoint.sh +++ b/containers/caddy/entrypoint.sh @@ -7,16 +7,20 @@ PYTHON="python3" JSON_DIR="${JSON_DIR:-/caddy_json_export}" CADDYFILE_PATH="${CADDYFILE_PATH:-/etc/caddy/Caddyfile}" CADDY_PID="" +CLUSTER_NODE="${CLUSTER_NODE:-false}" -echo "==> Generating wireguard_webadmin.json..." -"$PYTHON" "${SCRIPT_DIR}/export_wireguard_webadmin_config.py" +# The WireGuard WebAdmin interface is only published by the primary server +if [ "$CLUSTER_NODE" != "true" ]; then + echo "==> Generating wireguard_webadmin.json..." + "$PYTHON" "${SCRIPT_DIR}/export_wireguard_webadmin_config.py" +fi generate_caddyfile() { - "$PYTHON" "${SCRIPT_DIR}/process_config.py" + "$PYTHON" "${SCRIPT_DIR}/process_config.py" >/dev/null # Validation output is not printed because provider errors may include credentials if ! caddy validate --config "$CADDYFILE_PATH" --adapter caddyfile >/dev/null 2>&1; then echo "Warning: generated Caddyfile is invalid, regenerating with DNS challenge disabled. Check the DNS credentials." - DISABLE_DNS_CHALLENGE=1 "$PYTHON" "${SCRIPT_DIR}/process_config.py" + DISABLE_DNS_CHALLENGE=1 "$PYTHON" "${SCRIPT_DIR}/process_config.py" >/dev/null fi } @@ -42,27 +46,63 @@ stop_caddy() { echo "==> Processing config files..." generate_caddyfile sync_tls_state >/dev/null +# Nothing is cached yet, so hosts left on a fallback certificate get a new ACME attempt +"$PYTHON" "${SCRIPT_DIR}/process_config.py" --reset-fallback-certificates echo "==> Starting Caddy..." start_caddy +if [ "$CLUSTER_NODE" != "true" ]; then + # Certificates used by cluster workers are exported as soon as they are issued or renewed + ( + while true; do + sleep 60 + "$PYTHON" "${SCRIPT_DIR}/process_config.py" --export-certificates || true + done + ) & +fi + sleep 2 +caddyfile_checksum() { + sha256sum "$CADDYFILE_PATH" 2>/dev/null | cut -d ' ' -f 1 +} + +APPLIED_CADDYFILE="$(caddyfile_checksum)" + echo "==> Watching ${JSON_DIR} for config changes..." while true; do - inotifywait -qq -e close_write,moved_to,create "${JSON_DIR}/" --include '.*\.json$' 2>/dev/null || true + # The timeout also picks up changes written before the watch started or missed while reloading + inotifywait -qq -t 60 -e close_write,moved_to,create "${JSON_DIR}/" --include '.*\.json$' 2>/dev/null || true sleep 1 - echo "==> Config change detected, reprocessing..." + # The container that owns this network namespace was restarted: exit so Docker restarts Caddy and it + # joins the new namespace (otherwise it keeps serving from a namespace without published ports) + if [ -f "${JSON_DIR}/.netns_owner" ] && [ "$(cat "${JSON_DIR}/.netns_owner")" != "$(readlink /proc/self/ns/net)" ]; then + echo "Network namespace owner changed, restarting..." + stop_caddy + exit 1 + fi + generate_caddyfile - if [ "$(sync_tls_state)" = "restart" ]; then + if ! kill -0 "$CADDY_PID" 2>/dev/null; then + echo "==> Caddy is not running, starting it again..." + sync_tls_state >/dev/null + start_caddy + APPLIED_CADDYFILE="$(caddyfile_checksum)" + elif [ "$(sync_tls_state)" = "restart" ]; then # A reload keeps previously issued certificates cached in memory echo "==> Certificate settings changed, restarting Caddy..." stop_caddy start_caddy - else - echo "==> Reloading Caddy..." - caddy reload --config "$CADDYFILE_PATH" --adapter caddyfile 2>/dev/null || echo "Warning: Caddy reload failed, will retry on next change." + APPLIED_CADDYFILE="$(caddyfile_checksum)" + elif [ "$(caddyfile_checksum)" != "$APPLIED_CADDYFILE" ]; then + echo "==> Config change detected, reloading Caddy..." + if caddy reload --config "$CADDYFILE_PATH" --adapter caddyfile 2>/dev/null; then + APPLIED_CADDYFILE="$(caddyfile_checksum)" + else + echo "Warning: Caddy reload failed, will retry on next change." + fi fi done diff --git a/containers/caddy/process_config.py b/containers/caddy/process_config.py index 855864f..9daa6f9 100644 --- a/containers/caddy/process_config.py +++ b/containers/caddy/process_config.py @@ -36,6 +36,10 @@ TLS_STATE_PATH = os.path.join(CADDY_DATA_DIR, "wireguard_webadmin_tls_state.json TLS_STATE_PENDING_PATH = f"{CADDYFILE_PATH}.tls_state.json" # Set by entrypoint.sh when the generated Caddyfile fails validation DISABLE_DNS_CHALLENGE = os.environ.get("DISABLE_DNS_CHALLENGE", "") == "1" +# Cluster nodes never request ACME certificates: they use the ones exported by the primary server +CLUSTER_NODE = os.environ.get("CLUSTER_NODE", "").lower() == "true" +NODE_CERTIFICATES_DIR = os.environ.get("NODE_CERTIFICATES_DIR", "/caddy_certificates") +CERTIFICATE_EXPORT_DIR = os.environ.get("CERTIFICATE_EXPORT_DIR", "/caddy_cert_export") AUTH_IDENTITY_HEADERS = ( "X-Auth-User", "X-Auth-Email", @@ -92,12 +96,30 @@ def _safe_identifier(val: str) -> str: return re.sub(r"[^A-Za-z0-9_]", "", str(val)) +def get_node_host_tls(host_tls): + """ + TLS settings of a host on a cluster node: the certificate exported by the primary server, the internal CA + as fallback, or None when there is no certificate yet and the fallback is disabled. + """ + certificate_type = host_tls.get("certificate_type", DEFAULT_CERTIFICATE_TYPE) + certificate_fallback = host_tls.get("certificate_fallback", True) is not False + certificate_file = re.sub(r"[^a-f0-9]", "", str(host_tls.get("certificate_file", ""))) + if certificate_type in ACME_CERTIFICATE_TYPES and certificate_file: + return "certificate_file", certificate_file, certificate_fallback + if certificate_type == "self_signed" or certificate_fallback: + return "self_signed", None, True + return None + + def get_host_tls(host, tls_data): """ Return (certificate_type, dns_credential_name, certificate_fallback) for a host. - Hosts not listed use the HTTP challenge with fallback. + Hosts not listed use the HTTP challenge with fallback. On cluster nodes the second item is the + certificate file name, and None is returned for hosts that must not be served. """ host_tls = tls_data.get("hosts", {}).get(host.lower(), {}) + if CLUSTER_NODE: + return get_node_host_tls(host_tls) certificate_type = host_tls.get("certificate_type", DEFAULT_CERTIFICATE_TYPE) if certificate_type not in CERTIFICATE_TYPES: certificate_type = DEFAULT_CERTIFICATE_TYPE @@ -117,15 +139,29 @@ def group_hosts_by_tls(hosts, tls_data): """Hosts sharing the same certificate settings are served by the same site block.""" groups = {} for host in hosts: - groups.setdefault(get_host_tls(host, tls_data), []).append(host) + host_tls = get_host_tls(host, tls_data) + if host_tls is not None: + groups.setdefault(host_tls, []).append(host) return groups.items() +def get_certificate_only_hosts(apps, tls_data): + """Hosts the primary server only manages certificates for, because their application is served by workers.""" + if CLUSTER_NODE: + return [] + served_hosts = {host.lower() for app in apps for host in app.get("hosts", [])} + return [ + host for host, host_tls in tls_data.get("hosts", {}).items() + if host_tls.get("certificate_only") and host not in served_hosts + ] + + def build_caddyfile(apps, auth_policies, routes, tls_data=None): tls_data = tls_data or {} policies = auth_policies.get("policies", {}) if auth_policies else {} route_entries = routes.get("entries", {}) if routes else {} - lines = [] + # Caddy refuses an empty Caddyfile, which happens on cluster nodes without applications + lines = ["# Generated by process_config.py. Do not edit.", ""] def get_policy_type(policy_name): if policy_name and policy_name in policies: @@ -144,6 +180,12 @@ def build_caddyfile(apps, auth_policies, routes, tls_data=None): lines.append("") return + if certificate_type == "certificate_file": + certificate_path = os.path.join(NODE_CERTIFICATES_DIR, dns_credential) + lines.append(f" tls {certificate_path}.crt {certificate_path}.key") + lines.append("") + return + lines.append(" tls {") if certificate_type == "dns_challenge": dns_credential_data = tls_data["dns_credentials"][dns_credential] @@ -299,6 +341,15 @@ def build_caddyfile(apps, auth_policies, routes, tls_data=None): for (certificate_type, dns_credential, certificate_fallback), group_hosts in group_hosts_by_tls(hosts, tls_data): emit_site_block(app, group_hosts, certificate_type, dns_credential, certificate_fallback) + certificate_only_hosts = get_certificate_only_hosts(apps, tls_data) + for (certificate_type, dns_credential, certificate_fallback), group_hosts in group_hosts_by_tls(certificate_only_hosts, tls_data): + # Keeps the certificate managed for cluster workers; the application is not published here + lines.append(f"{', '.join(_safe_value(host) for host in group_hosts)} {{") + emit_tls(certificate_type, dns_credential, certificate_fallback) + lines.append(" respond 404") + lines.append("}") + lines.append("") + return "\n".join(lines) @@ -306,19 +357,21 @@ def build_tls_state(apps, tls_data): """Map each host to a fingerprint of the certificate settings used in the generated Caddyfile.""" tls_data = tls_data or {} tls_state = {} - for app in apps: - if not app.get("hosts") or not app.get("upstream"): + hosts = [host for app in apps if app.get("upstream") for host in app.get("hosts", [])] + hosts += get_certificate_only_hosts(apps, tls_data) + for host in hosts: + host_tls = get_host_tls(host, tls_data) + if host_tls is None: continue - for host in app["hosts"]: - certificate_type, dns_credential, certificate_fallback = get_host_tls(host, tls_data) - credential_hash = "" - if dns_credential: - credential_data = json.dumps(tls_data["dns_credentials"][dns_credential], sort_keys=True) - credential_hash = hashlib.sha256(credential_data.encode()).hexdigest() - certificate_generation = tls_data.get("hosts", {}).get(host.lower(), {}).get("certificate_generation", 0) - tls_state[host.lower()] = [ - certificate_type, dns_credential or "", credential_hash, certificate_fallback, certificate_generation - ] + certificate_type, certificate_reference, certificate_fallback = host_tls + credential_hash = "" + if certificate_type == "dns_challenge": + credential_data = json.dumps(tls_data["dns_credentials"][certificate_reference], sort_keys=True) + credential_hash = hashlib.sha256(credential_data.encode()).hexdigest() + certificate_generation = tls_data.get("hosts", {}).get(host.lower(), {}).get("certificate_generation", 0) + tls_state[host.lower()] = [ + certificate_type, certificate_reference or "", credential_hash, certificate_fallback, certificate_generation + ] return tls_state @@ -363,9 +416,22 @@ def sync_tls_state(): restart_required = False if applied_state is not None: for host, host_state in pending_state.items(): - if applied_state.get(host) in (None, host_state): + previous_host_state = applied_state.get(host) + if previous_host_state == host_state or "/" in host or host in ("", ".", ".."): continue - if "/" in host or host in ("", ".", ".."): + # A new host normally has nothing cached, unless a fallback certificate was left by an earlier use + # of the same hostname (e.g. removed from one application and added to another) + if previous_host_state is None and not os.path.isdir( + os.path.join(CERTIFICATES_DIR, INTERNAL_ISSUER_DIR_NAME, host) + ): + continue + if CLUSTER_NODE: + # Certificates received from the primary server replace cached ones only after a restart + if host_state[0] == "certificate_file": + fallback_certificate_dir = os.path.join(CERTIFICATES_DIR, INTERNAL_ISSUER_DIR_NAME, host) + if os.path.isdir(fallback_certificate_dir): + shutil.rmtree(fallback_certificate_dir) + restart_required = True continue if host_state[0] not in ACME_CERTIFICATE_TYPES: # A cached ACME certificate would keep being served instead of the self-signed one @@ -386,6 +452,90 @@ def sync_tls_state(): print("restart") +def reset_fallback_certificates(): + """ + Remove fallback certificates of ACME hosts before Caddy starts, so the ACME certificate is requested again + on every start instead of only on the next renewal. If the request fails, the fallback is issued again. + """ + pending_state = _read_json(TLS_STATE_PENDING_PATH) or {} + for host, host_state in pending_state.items(): + if host_state[0] not in ACME_CERTIFICATE_TYPES or "/" in host or host in ("", ".", ".."): + continue + fallback_certificate_dir = os.path.join(CERTIFICATES_DIR, INTERNAL_ISSUER_DIR_NAME, host) + if os.path.isdir(fallback_certificate_dir): + shutil.rmtree(fallback_certificate_dir) + print(f"Removed fallback certificate for {host}", file=sys.stderr) + + +def find_acme_certificate(host): + """Return (certificate_path, key_path) of the newest ACME certificate stored by Caddy for a host.""" + candidates = [] + if os.path.isdir(CERTIFICATES_DIR): + for issuer_dir_name in os.listdir(CERTIFICATES_DIR): + if issuer_dir_name == INTERNAL_ISSUER_DIR_NAME: + continue + certificate_path = os.path.join(CERTIFICATES_DIR, issuer_dir_name, host, f"{host}.crt") + key_path = os.path.join(CERTIFICATES_DIR, issuer_dir_name, host, f"{host}.key") + if os.path.isfile(certificate_path) and os.path.isfile(key_path): + candidates.append((os.path.getmtime(certificate_path), certificate_path, key_path)) + if not candidates: + return None, None + newest = max(candidates) + return newest[1], newest[2] + + +def _copy_if_changed(source_path, destination_path): + with open(source_path, "r", encoding="utf-8") as source_file: + content = source_file.read() + if _read_text(destination_path) == content: + return False + temp_path = f"{destination_path}.tmp" + with open(temp_path, "w", encoding="utf-8") as destination_file: + destination_file.write(content) + os.chmod(temp_path, 0o600) + os.replace(temp_path, destination_path) + return True + + +def _read_text(filepath): + try: + with open(filepath, "r", encoding="utf-8") as text_file: + return text_file.read() + except OSError: + return None + + +def export_certificates(): + """ + Copy the ACME certificates of hosts served by cluster workers to CERTIFICATE_EXPORT_DIR, where Django reads + them to distribute to the workers. Fallback certificates from the internal CA are never exported. + """ + if CLUSTER_NODE or not os.path.isdir(CERTIFICATE_EXPORT_DIR): + return + tls_data = load_json("tls.json") or {} + export_hosts = { + host for host, host_tls in tls_data.get("hosts", {}).items() + if host_tls.get("export_certificate") and "/" not in host and host not in ("", ".", "..") + } + + for host in sorted(export_hosts): + certificate_path, key_path = find_acme_certificate(host) + if not certificate_path: + continue + host_export_dir = os.path.join(CERTIFICATE_EXPORT_DIR, host) + os.makedirs(host_export_dir, exist_ok=True) + key_changed = _copy_if_changed(key_path, os.path.join(host_export_dir, "certificate.key")) + certificate_changed = _copy_if_changed(certificate_path, os.path.join(host_export_dir, "certificate.crt")) + if certificate_changed or key_changed: + print(f"Exported certificate for {host}") + + for host in os.listdir(CERTIFICATE_EXPORT_DIR): + host_export_dir = os.path.join(CERTIFICATE_EXPORT_DIR, host) + if host not in export_hosts and os.path.isdir(host_export_dir): + shutil.rmtree(host_export_dir) + print(f"Removed exported certificate for {host}") + + def main(): apps = collect_all_applications() auth_policies = load_json("auth_policies.json") @@ -403,5 +553,9 @@ def main(): if __name__ == "__main__": if "--sync-tls-state" in sys.argv: sync_tls_state() + elif "--export-certificates" in sys.argv: + export_certificates() + elif "--reset-fallback-certificates" in sys.argv: + reset_fallback_certificates() else: main() diff --git a/containers/cluster_node/Dockerfile-cluster_node b/containers/cluster_node/Dockerfile-cluster_node index 532af55..b34ff6a 100644 --- a/containers/cluster_node/Dockerfile-cluster_node +++ b/containers/cluster_node/Dockerfile-cluster_node @@ -12,6 +12,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ inetutils-traceroute \ procps \ curl \ + openssl \ nano \ && rm -rf /var/lib/apt/lists/* diff --git a/containers/cluster_node/cluster_worker.py b/containers/cluster_node/cluster_worker.py index b441036..49f09ea 100644 --- a/containers/cluster_node/cluster_worker.py +++ b/containers/cluster_node/cluster_worker.py @@ -1,350 +1,432 @@ import glob +import json import logging import os +import re +import socket +import ssl import subprocess +import tempfile import time -from typing import Dict, Any +from datetime import datetime, timezone import requests -# Configure logging -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s' -) +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) -# Constants -MASTER_SERVER_ADDRESS = os.environ.get('MASTER_SERVER_ADDRESS') -TOKEN = os.environ.get('TOKEN') +MASTER_SERVER_ADDRESS = os.environ.get('MASTER_SERVER_ADDRESS', '') +TOKEN = os.environ.get('TOKEN', '') +WORKER_VERSION = 12 +REQUEST_TIMEOUT = 30 +MANIFEST_INTERVAL = 60 +DEFAULT_STATS_INTERVAL = 60 + WIREGUARD_DIR = '/etc/wireguard' +FIREWALL_SCRIPT = os.path.join(WIREGUARD_DIR, 'wg-firewall.sh') DNS_DIR = '/etc/dnsmasq' -WORKER_VERSION = 11 -REQUEST_TIMEOUT = 20 - -class ClusterWorker: - def __init__(self): - self.config_version = 0 - self.dns_version = self.get_local_dns_version() - self.should_run = True - self.session = requests.Session() - - if not MASTER_SERVER_ADDRESS or not TOKEN: - logger.error("MASTER_SERVER_ADDRESS or TOKEN not set") - self.should_run = False - - self.base_url = f"https://{MASTER_SERVER_ADDRESS}/api/cluster" - # Validate URL scheme - if not MASTER_SERVER_ADDRESS.startswith(('http://', 'https://')): - self.base_url = f"https://{MASTER_SERVER_ADDRESS}/api/cluster" - else: - self.base_url = f"{MASTER_SERVER_ADDRESS}/api/cluster" +DNS_MAIN_CONFIG = 'wireguard_webadmin_dns.conf' +# Only mounted when the node runs the App Gateway (cluster_node-gateway.yml) +GATEWAY_DIR = '/caddy_json_export' +CERTIFICATES_DIR = '/caddy_certificates' +GATEWAY_FILES = ('applications.json', 'auth_policies.json', 'routes.json', 'tls.json') +CERTIFICATE_FILE_PATTERN = re.compile(r'^[a-f0-9]{64}\.(crt|key)$') +CADDY_ADMIN_URL = 'http://127.0.0.1:2019/config/' +AUTH_GATEWAY_URL = 'http://cluster-node-auth-gateway:9091/' +INTERNAL_CA_NAME = 'Caddy Local Authority' +FIREWALL_CHAINS = ( + ('nat', 'WGWADM_POSTROUTING'), + ('nat', 'WGWADM_PREROUTING'), + ('filter', 'WGWADM_FORWARD'), + ('filter', 'WGWADM_ROUTE_POLICY'), +) - def func_process_wireguard_status(self) -> Dict[str, Any]: - command = "wg show all dump" - - process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - stdout, stderr = process.communicate() - - if process.returncode != 0: - return {"message": stderr, "status": "error"} - - data: Dict[str, Dict[str, Dict[str, Any]]] = {} - - # wg dump format is tab-separated. - # There are two kinds of lines: - # - Interface line: interface \t private_key \t public_key \t listen_port \t fwmark - # - Peer line: interface \t peer_public_key \t preshared_key \t endpoint \t allowed_ips \t latest_handshake \t transfer_rx \t transfer_tx \t persistent_keepalive - for line in stdout.strip().split("\n"): - parts = line.split("\t") - if len(parts) < 5: - continue - - interface = parts[0] - - # Peer lines are expected to have at least 9 fields - if len(parts) < 9: - # interface line, ignore - continue - - peer_public_key = parts[1] - endpoint = parts[3] - allowed_ips_raw = parts[4] - latest_handshake = parts[5] - transfer_rx = parts[6] - transfer_tx = parts[7] - - if interface not in data: - data[interface] = {} - - data[interface][peer_public_key] = { - "allowed-ips": [ip for ip in allowed_ips_raw.split(",") if ip] if allowed_ips_raw else [], - "latest-handshakes": latest_handshake or "", - "transfer": {"tx": int(transfer_tx or 0), "rx": int(transfer_rx or 0)}, - "endpoints": endpoint or "", - } - - return data +class SectionError(Exception): + pass - def cleanup_wireguard(self): - logger.info("Cleaning up WireGuard configuration...") - # Stop all wireguard interfaces - try: - files = glob.glob(f"{WIREGUARD_DIR}/*.conf") - for f in files: - interface = os.path.basename(f).replace('.conf', '') - subprocess.run(['wg-quick', 'down', interface], capture_output=True) - except Exception as e: - logger.error(f"Error stopping wireguard interfaces: {e}") +def run_command(command, check=True): + result = subprocess.run(command, capture_output=True, text=True) + if check and result.returncode != 0: + raise SectionError(f"{' '.join(command)}: {(result.stderr or result.stdout).strip()}") + return result - # Remove files - try: - for f in glob.glob(f"{WIREGUARD_DIR}/*"): - os.remove(f) - except Exception as e: - logger.error(f"Error cleaning directory: {e}") - try: - subprocess.run(['iptables', '-t', 'nat', '-F', 'WGWADM_POSTROUTING'], capture_output=True) - subprocess.run(['iptables', '-t', 'nat', '-F', 'WGWADM_PREROUTING'], capture_output=True) - subprocess.run(['iptables', '-t', 'filter', '-F', 'WGWADM_FORWARD'], capture_output=True) - except Exception as e: - logger.error(f"Error flushing firewall: {e}") +def interface_is_up(interface): + return run_command(['wg', 'show', interface], check=False).returncode == 0 - def get_local_dns_version(self): - try: - version_file = os.path.join(DNS_DIR, 'config_version.conf') - if os.path.exists(version_file): - with open(version_file, 'r') as f: - for line in f: - if line.startswith('DNS_VERSION='): - return int(line.strip().split('=')[1]) - except Exception as e: - logger.error(f"Error reading DNS version: {e}") - return 0 - def get_status(self): - params = { - 'token': TOKEN, - 'worker_config_version': self.config_version, - 'worker_dns_version': self.dns_version, - 'worker_version': WORKER_VERSION +def interface_section(config): + """Return the [Interface] part of a WireGuard config, used to decide between syncconf and restart.""" + return config.split('[Peer]', 1)[0].strip() + + +def read_file(path): + try: + with open(path, 'r', encoding='utf-8') as config_file: + return config_file.read() + except FileNotFoundError: + return None + + +def write_file(path, content, mode=0o600, in_place=False): + if in_place: + with open(path, 'w', encoding='utf-8') as config_file: + config_file.write(content) + os.chmod(path, mode) + return + temp_path = f'{path}.tmp' + with open(temp_path, 'w', encoding='utf-8') as config_file: + config_file.write(content) + os.chmod(temp_path, mode) + os.replace(temp_path, path) + + +def wireguard_status(): + result = run_command(['wg', 'show', 'all', 'dump'], check=False) + if result.returncode != 0: + return {} + + data = {} + # Peer lines: interface, public_key, preshared_key, endpoint, allowed_ips, latest_handshake, rx, tx, keepalive + for line in result.stdout.strip().splitlines(): + parts = line.split('\t') + if len(parts) < 9: + continue + interface, peer_public_key, endpoint, allowed_ips = parts[0], parts[1], parts[3], parts[4] + data.setdefault(interface, {})[peer_public_key] = { + 'allowed-ips': [ip for ip in allowed_ips.split(',') if ip] if allowed_ips else [], + 'latest-handshakes': parts[5] or '', + 'transfer': {'tx': int(parts[7] or 0), 'rx': int(parts[6] or 0)}, + 'endpoints': endpoint or '', } - logger.info(f"Requesting status from Master... (Config Version: {self.config_version}, DNS Version: {self.dns_version}, Worker Version: {WORKER_VERSION})") - try: - response = self.session.get(f"{self.base_url}/status/", params=params, timeout=REQUEST_TIMEOUT) - logger.info(f"Status response received. HTTP Code: {response.status_code}") - return response - except requests.RequestException as e: - logger.error(f"Connection error while getting status: {e}") - return None + return data - def download_configs(self): - params = { - 'token': TOKEN, - 'worker_config_version': self.config_version, - 'worker_dns_version': self.dns_version, - 'worker_version': WORKER_VERSION + +def gateway_enabled(): + return os.path.isdir(GATEWAY_DIR) and os.path.isdir(CERTIFICATES_DIR) + + +def http_service_is_running(url): + try: + requests.get(url, timeout=3) + return True + except requests.RequestException: + return False + + +def _parse_not_after(value): + return datetime.strptime(value, '%b %d %H:%M:%S %Y %Z').replace(tzinfo=timezone.utc).isoformat() + + +def _fetch_certificate(hostname, verify): + if verify: + context = ssl.create_default_context() + else: + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + with socket.create_connection(('127.0.0.1', 443), timeout=3) as raw_socket: + with context.wrap_socket(raw_socket, server_hostname=hostname) as tls_socket: + return tls_socket.getpeercert() if verify else tls_socket.getpeercert(binary_form=True) + + +def inspect_certificate(hostname): + """Same checks as app_gateway/certificate_status.py on the primary server, against the local Caddy.""" + try: + certificate = _fetch_certificate(hostname, verify=True) + issuer = dict(field[0] for field in certificate.get('issuer', ())) + return { + 'status': 'trusted', + 'issuer': issuer.get('organizationName') or issuer.get('commonName') or '', + 'not_after': _parse_not_after(certificate['notAfter']), } - try: - response = self.session.get(f"{self.base_url}/worker/get_config_files/", params=params, timeout=REQUEST_TIMEOUT) - if response.status_code == 200: - data = response.json() - if data.get('status') == 'success': - return data - return None - except requests.RequestException as e: - logger.error(f"Error downloading configs: {e}") - return None + except ssl.SSLCertVerificationError: + pass + except (OSError, ValueError): + return {'status': 'unavailable', 'issuer': '', 'not_after': None} - def download_dns_config(self): - params = { - 'token': TOKEN, - 'worker_config_version': self.config_version, - 'worker_dns_version': self.dns_version, - 'worker_version': WORKER_VERSION - } - try: - logger.info(f"Downloading DNS config... (Current Version: {self.dns_version})") - response = self.session.get(f"{self.base_url}/worker/get_dnsmasq_config/", params=params, timeout=REQUEST_TIMEOUT) - if response.status_code == 200: - tar_path = 'dnsmasq_config.tar.gz' - with open(tar_path, 'wb') as f: - f.write(response.content) - - logger.info("Extracting DNS config...") - if not os.path.exists(DNS_DIR): - os.makedirs(DNS_DIR, exist_ok=True) - - subprocess.run(['tar', 'xvfz', tar_path, '-C', DNS_DIR], check=True, capture_output=True) - if os.path.exists(tar_path): - os.remove(tar_path) - - # Update version - self.dns_version = self.get_local_dns_version() - logger.info(f"DNS config updated (New Version: {self.dns_version})") - return True - else: - logger.error(f"Failed to download DNS config. Status: {response.status_code}") - return False - except Exception as e: - logger.error(f"Error updating DNS config: {e}") - return False + try: + certificate_der = _fetch_certificate(hostname, verify=False) + result = subprocess.run( + ['openssl', 'x509', '-inform', 'DER', '-noout', '-issuer', '-enddate', '-nameopt', 'RFC2253'], + input=certificate_der, capture_output=True, timeout=5, + ) + except (OSError, ValueError, subprocess.SubprocessError): + return {'status': 'unavailable', 'issuer': '', 'not_after': None} + issuer = '' + not_after = None + for line in result.stdout.decode(errors='replace').splitlines(): + if line.startswith('issuer='): + parts = dict(part.split('=', 1) for part in line[len('issuer='):].split(',') if '=' in part) + issuer = INTERNAL_CA_NAME if INTERNAL_CA_NAME in line else parts.get('O') or parts.get('CN') or '' + elif line.startswith('notAfter='): + not_after = _parse_not_after(line[len('notAfter='):]) + return {'status': 'untrusted', 'issuer': issuer, 'not_after': not_after} - def send_ping(self): - params = { - 'token': TOKEN, - 'worker_config_version': self.config_version, - 'worker_dns_version': self.dns_version, - 'worker_version': WORKER_VERSION - } - try: - logger.info("Sending ping to Master...") - self.session.get(f"{self.base_url}/worker/ping/", params=params, timeout=REQUEST_TIMEOUT) - except Exception as e: - logger.error(f"Error sending ping: {e}") - def apply_configs(self, data): - logger.info("Applying new configurations...") - files = data.get('files', {}) - cluster_settings = data.get('cluster_settings', {}) - new_config_version = cluster_settings.get('config_version', 0) - - # 1. Stop existing interfaces - self.cleanup_wireguard() - - # 2. Write new files - for filename, content in files.items(): - if filename == 'wg-firewall.sh' and isinstance(content, str): - content = content.replace('wireguard-webadmin-dns', 'cluster-node-dns') - filepath = os.path.join(WIREGUARD_DIR, filename) - with open(filepath, 'w') as f: - f.write(content) - - if filename == 'wg-firewall.sh': - os.chmod(filepath, 0o755) - - # Start interfaces - conf_files = glob.glob(f"{WIREGUARD_DIR}/*.conf") - for conf in conf_files: - interface = os.path.basename(conf).replace('.conf', '') - logger.info(f"Starting WireGuard interface: {interface}") +def dns_is_running(): + # dnsmasq always listens on loopback when it is restricted to specific interfaces. The DNS container pauses + # for a few seconds while reloading, so a missing answer is retried before reporting it as stopped. + query = b'\x12\x34\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x09localhost\x00\x00\x01\x00\x01' + for attempt in range(4): + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as dns_socket: + dns_socket.settimeout(2) try: - subprocess.run(['wg-quick', 'up', interface], check=True, capture_output=True) - except subprocess.CalledProcessError as e: - logger.error(f"Failed to start {interface}: {e.stderr.decode()}") + dns_socket.sendto(query, ('127.0.0.1', 53)) + dns_socket.recvfrom(512) + return True + except OSError: + pass + time.sleep(1) + return False - # 4. Update config version - self.config_version = new_config_version - logger.info(f"Configuration updated to version {self.config_version}") + +class ClusterNodeAgent: + def __init__(self): + self.base_url = f"{MASTER_SERVER_ADDRESS.rstrip('/')}/api/cluster/v2" + self.session = requests.Session() + self.session.headers.update({ + 'Authorization': f'Bearer {TOKEN}', + 'X-Worker-Version': str(WORKER_VERSION), + }) + # Nothing is persisted: after a restart interfaces and firewall rules are gone, so every section is applied again + self.applied_hashes = {} + self.stats_interval = DEFAULT_STATS_INTERVAL + self.wireguard_active = False + + def request(self, method, path, **kwargs): + try: + return self.session.request(method, f'{self.base_url}/{path}', timeout=REQUEST_TIMEOUT, **kwargs) + except requests.RequestException as error: + logger.error('Request to %s failed: %s', path, error) + return None + + # Section handlers + + def apply_firewall(self, section): + content = section['files'].get('wg-firewall.sh') + if content is None: + raise SectionError('wg-firewall.sh missing from section') + write_file(FIREWALL_SCRIPT, content, mode=0o700) + run_command(['bash', FIREWALL_SCRIPT]) + + def apply_wireguard(self, section): + desired_files = { + filename: content for filename, content in section['files'].items() + if filename.startswith('wg') and filename.endswith('.conf') and '/' not in filename + } + errors = [] + + for config_path in glob.glob(os.path.join(WIREGUARD_DIR, 'wg[0-9]*.conf')): + filename = os.path.basename(config_path) + if filename in desired_files: + continue + interface = filename[:-len('.conf')] + if interface_is_up(interface): + run_command(['wg-quick', 'down', interface], check=False) + os.remove(config_path) + logger.info('Removed WireGuard interface %s', interface) + + for filename, content in sorted(desired_files.items()): + interface = filename[:-len('.conf')] + config_path = os.path.join(WIREGUARD_DIR, filename) + previous_content = read_file(config_path) + write_file(config_path, content) + try: + if not interface_is_up(interface): + run_command(['wg-quick', 'up', interface]) + logger.info('Started WireGuard interface %s', interface) + elif previous_content is None or interface_section(previous_content) != interface_section(content): + run_command(['wg-quick', 'down', interface], check=False) + run_command(['wg-quick', 'up', interface]) + logger.info('Restarted WireGuard interface %s', interface) + elif previous_content != content: + self.sync_peers(interface, config_path) + logger.info('Synchronized peers of WireGuard interface %s', interface) + except SectionError as error: + errors.append(str(error)) + + self.wireguard_active = True + if errors: + raise SectionError('; '.join(errors)) + + @staticmethod + def sync_peers(interface, config_path): + stripped_config = run_command(['wg-quick', 'strip', config_path]).stdout + with tempfile.NamedTemporaryFile('w', suffix='.conf', delete=False) as stripped_file: + stripped_file.write(stripped_config) + try: + run_command(['wg', 'syncconf', interface, stripped_file.name]) + finally: + os.remove(stripped_file.name) + + def apply_dns(self, section): + files = section['files'] + if DNS_MAIN_CONFIG not in files: + raise SectionError(f'{DNS_MAIN_CONFIG} missing from section') + os.makedirs(DNS_DIR, exist_ok=True) + for filename, content in files.items(): + if filename != DNS_MAIN_CONFIG and '/' not in filename: + write_file(os.path.join(DNS_DIR, filename), content, mode=0o644) + for config_path in glob.glob(os.path.join(DNS_DIR, '*.conf')): + if os.path.basename(config_path) not in files: + os.remove(config_path) + # Written last and in place: the DNS container watches this file with inotify to reload + write_file(os.path.join(DNS_DIR, DNS_MAIN_CONFIG), files[DNS_MAIN_CONFIG], mode=0o644, in_place=True) + + def apply_settings(self, section): + self.stats_interval = int(section['values'].get('stats_sync_interval', DEFAULT_STATS_INTERVAL)) + + def apply_certificates(self, section): + if not gateway_enabled(): + return + files = { + filename: content for filename, content in section['files'].items() + if CERTIFICATE_FILE_PATTERN.match(filename) + } + for filename, content in files.items(): + certificate_path = os.path.join(CERTIFICATES_DIR, filename) + if read_file(certificate_path) != content: + write_file(certificate_path, content, mode=0o600) + for certificate_path in glob.glob(os.path.join(CERTIFICATES_DIR, '*')): + if os.path.basename(certificate_path) not in files: + os.remove(certificate_path) + + def apply_gateway(self, section): + if not gateway_enabled(): + return + for filename in GATEWAY_FILES: + if filename not in section['files']: + raise SectionError(f'{filename} missing from section') + for filename in GATEWAY_FILES: + write_file(os.path.join(GATEWAY_DIR, filename), section['files'][filename], mode=0o644) + + SECTION_HANDLERS = { + 'firewall': apply_firewall, + 'wireguard': apply_wireguard, + 'dns': apply_dns, + 'settings': apply_settings, + 'certificates': apply_certificates, + 'gateway': apply_gateway, + } + + # Main loop + + def shutdown_wireguard(self): + if not self.wireguard_active: + return + logger.warning('Shutting down WireGuard interfaces and firewall rules') + for config_path in glob.glob(os.path.join(WIREGUARD_DIR, 'wg[0-9]*.conf')): + run_command(['wg-quick', 'down', os.path.basename(config_path)[:-len('.conf')]], check=False) + for table, chain in FIREWALL_CHAINS: + run_command(['iptables', '-t', table, '-F', chain], check=False) + self.applied_hashes.pop('wireguard', None) + self.applied_hashes.pop('firewall', None) + self.wireguard_active = False + + def sync_configuration(self): + response = self.request('GET', 'manifest/') + if response is None: + return True + if response.status_code == 403: + logger.error('Token rejected by the primary server. Stopping.') + self.shutdown_wireguard() + return False + if response.status_code == 400: + logger.warning('Primary server refused this worker: %s', response.json().get('message', '')) + self.shutdown_wireguard() + return True + if response.status_code != 200: + logger.error('Unexpected manifest response: HTTP %s', response.status_code) + return True + + manifest = response.json().get('sections', {}) + reports = {} + for section_name, section_hash in manifest.items(): + if section_name not in self.SECTION_HANDLERS or self.applied_hashes.get(section_name) == section_hash: + continue + section_response = self.request('GET', f'section/{section_name}/') + if section_response is None or section_response.status_code != 200: + continue + section = section_response.json() + try: + self.SECTION_HANDLERS[section_name](self, section) + self.applied_hashes[section_name] = section['hash'] + reports[section_name] = {'hash': section['hash'], 'status': 'ok'} + logger.info('Section %s applied', section_name) + except (SectionError, OSError, ValueError) as error: + # Forget the last applied content: the next version must be applied and reported even if it + # is identical to the one that was applied before the failure + self.applied_hashes.pop(section_name, None) + reports[section_name] = {'hash': section['hash'], 'status': 'error', 'message': str(error)} + logger.error('Section %s failed: %s', section_name, error) + + if reports: + # Services are reported with the periodic stats: right after a DNS change dnsmasq is still reloading + self.request('POST', 'report/', json={'sections': reports}) + return True + + @staticmethod + def services_status(): + services = {'dnsmasq': 'running' if dns_is_running() else 'stopped'} + if gateway_enabled(): + services['caddy'] = 'running' if http_service_is_running(CADDY_ADMIN_URL) else 'stopped' + services['auth_gateway'] = 'running' if http_service_is_running(AUTH_GATEWAY_URL) else 'stopped' + return services + + @staticmethod + def certificates_status(): + try: + with open(os.path.join(GATEWAY_DIR, 'tls.json'), 'r', encoding='utf-8') as tls_file: + hosts = json.load(tls_file).get('hosts', {}) + except (OSError, ValueError): + return {} + return {hostname: inspect_certificate(hostname) for hostname in sorted(hosts)} def send_stats(self): - try: - stats = self.func_process_wireguard_status() - params = { - 'token': TOKEN, - 'worker_config_version': self.config_version, - 'worker_dns_version': self.dns_version, - 'worker_version': WORKER_VERSION - } - logger.info("Sending WireGuard stats to Master...") - response = self.session.post(f"{self.base_url}/worker/submit_wireguard_stats/", json=stats, params=params, timeout=REQUEST_TIMEOUT) - - if response.status_code == 200: - logger.info("Stats sent successfully.") - return True - else: - logger.error(f"Failed to send stats. Status: {response.status_code}") - return False - except Exception as e: - logger.error(f"Error sending stats: {e}") - return False + report = { + 'wireguard_status': wireguard_status(), + 'services': self.services_status(), + } + if gateway_enabled(): + report['certificates'] = self.certificates_status() + self.request('POST', 'report/', json=report) + + @staticmethod + def publish_network_namespace(): + # Containers sharing this network namespace (DNS, Caddy) restart themselves when it changes + network_namespace = os.readlink('/proc/self/ns/net') + for directory in (DNS_DIR, GATEWAY_DIR): + if os.path.isdir(directory): + write_file(os.path.join(directory, '.netns_owner'), network_namespace + '\n', mode=0o644) def run(self): - if not self.should_run: - return - - logger.info("Cluster Worker starting...") - - # Initial cleanup - self.cleanup_wireguard() - - last_config_check = 0 - last_stats_send = 0 - stats_sync_interval = 60 # Default initial value + logger.info('Cluster node agent %s starting', WORKER_VERSION) + self.publish_network_namespace() + self.wireguard_active = True + self.shutdown_wireguard() + last_manifest = 0 + last_stats = 0 while True: - current_time = time.time() - - # Check Config (Fixed 60s interval) - if current_time - last_config_check >= 60: - try: - response = self.get_status() - last_config_check = time.time() - - if response is not None: - if response.status_code == 403: - logger.error("Received 403 Forbidden (Token invalid/deleted). Deactivating WireGuard and stopping requests permanently.") - self.cleanup_wireguard() - self.config_version = 0 - self.should_run = False - break - - if response.status_code == 400: - logger.warning("Received 400 Bad Request (Worker suspended or error). Deactivating WireGuard and Firewall, but will keep retrying...") - self.cleanup_wireguard() - self.config_version = 0 - - if response.status_code == 200: - data = response.json() - - # Update stats interval from master settings - cluster_settings = data.get('cluster_settings', {}) - stats_sync_interval = cluster_settings.get('stats_sync_interval', 60) - - remote_config_version = cluster_settings.get('config_version', 0) - - # Check WireGuard Config - updated = False - if int(remote_config_version) != self.config_version: - logger.info(f"Config version mismatch (Local: {self.config_version}, Remote: {remote_config_version}). Updating...") - config_data = self.download_configs() - if config_data: - self.apply_configs(config_data) - self.send_ping() - updated = True - else: - logger.error("Failed to download config files.") - - # Check DNS Config - remote_dns_version = int(cluster_settings.get('dns_version', 0)) - if not updated and remote_dns_version != self.dns_version: - logger.info(f"DNS version mismatch (Local: {self.dns_version}, Remote: {remote_dns_version}). Updating...") - if self.download_dns_config(): - self.send_ping() - updated = True - - if not updated and int(remote_config_version) == self.config_version and remote_dns_version == self.dns_version: - logger.info(f"No changes detected. Configuration is up to date (WG: {self.config_version}, DNS: {self.dns_version}).") - - except Exception as e: - logger.error(f"Unexpected error in config check loop: {e}") - - # Check Stats Sync - if current_time - last_stats_send >= stats_sync_interval: + now = time.monotonic() + if now - last_manifest >= MANIFEST_INTERVAL or not last_manifest: + last_manifest = now + if not self.sync_configuration(): + break + if now - last_stats >= self.stats_interval: + last_stats = now self.send_stats() - last_stats_send = time.time() - - # Sleep briefly to be responsive but not busy loop time.sleep(1) - # Final loop state if 403 was received - while not self.should_run: - logger.info("Worker disabled due to 403 Forbidden. WireGuard is off.") - time.sleep(60) + while True: + time.sleep(3600) -if __name__ == "__main__": - worker = ClusterWorker() - worker.run() + +if __name__ == '__main__': + if not MASTER_SERVER_ADDRESS or not TOKEN: + logger.error('MASTER_SERVER_ADDRESS and TOKEN must be set') + else: + ClusterNodeAgent().run() diff --git a/containers/cluster_node/entrypoint.sh b/containers/cluster_node/entrypoint.sh index 297ceba..c4fc3da 100644 --- a/containers/cluster_node/entrypoint.sh +++ b/containers/cluster_node/entrypoint.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -if [[ "$COMPOSE_VERSION" != "c1b" ]]; then +if [[ "$COMPOSE_VERSION" != "c2a" ]]; then echo "ERROR: Please upgrade your docker compose file. Exiting." exit 1 fi diff --git a/containers/dnsmasq/entrypoint.sh b/containers/dnsmasq/entrypoint.sh index a188de7..e04f2c7 100644 --- a/containers/dnsmasq/entrypoint.sh +++ b/containers/dnsmasq/entrypoint.sh @@ -34,7 +34,24 @@ handle_sigint() { exit 0 } +# Written by cluster nodes, where this container shares the network namespace of the node. When the node +# container is restarted it gets a new namespace, so this container must restart to join it again. +NETNS_OWNER_FILE="/etc/dnsmasq/.netns_owner" + +watch_network_namespace() { + while sleep 30; do + if [ -f "$NETNS_OWNER_FILE" ] && [ "$(cat "$NETNS_OWNER_FILE")" != "$(readlink /proc/self/ns/net)" ]; then + echo "Network namespace of the cluster node changed, restarting..." + kill -TERM 1 + pkill inotifywait + return + fi + done +} + trap handle_sigint SIGINT +trap 'pkill dnsmasq; exit 1' SIGTERM create_default_config +watch_network_namespace & start_dnsmasq diff --git a/dns/functions.py b/dns/functions.py index e9d7002..2ff801f 100644 --- a/dns/functions.py +++ b/dns/functions.py @@ -1,58 +1,8 @@ import os -import tarfile -from cluster.models import ClusterSettings from .models import DNSSettings, StaticHost, DNSFilterList -def compress_dnsmasq_config(): - output_file = "/etc/dnsmasq/dnsmasq_config.tar.gz" - base_dir = "/etc/dnsmasq" - cluster_settings = ClusterSettings.objects.filter(enabled=True, name='cluster_settings').first() - if not cluster_settings: - if os.path.exists(output_file): - os.remove(output_file) - return None - - if not os.path.isdir(base_dir): - if os.path.exists(output_file): - os.remove(output_file) - return None - - conf_files = [ - fn for fn in os.listdir(base_dir) - if fn.endswith(".conf") and os.path.isfile(os.path.join(base_dir, fn)) - ] - - # If tar exists and is newer (or equal) than all .conf, do not recompile - if os.path.exists(output_file): - tar_mtime = os.path.getmtime(output_file) - newest_conf_mtime = max( - os.path.getmtime(os.path.join(base_dir, fn)) for fn in conf_files - ) - if newest_conf_mtime <= tar_mtime: - return output_file - - # If we reach here, we need to increment DNS version and recompile the tar.gz - cluster_settings.dns_version += 1 - cluster_settings.save() - dns_version_file = os.path.join(base_dir, "config_version.conf") - with open(dns_version_file, "w", encoding="utf-8") as f: - f.write(f"DNS_VERSION={cluster_settings.dns_version}\n") - if "config_version.conf" not in conf_files: - conf_files.append("config_version.conf") - - # Create tar.gz - tmp_output = output_file + ".tmp" - with tarfile.open(tmp_output, "w:gz") as tar: - for fn in conf_files: - fullpath = os.path.join(base_dir, fn) - tar.add(fullpath, arcname=fn) - - os.replace(tmp_output, output_file) - return output_file - - def generate_unbound_config(): dns_settings = DNSSettings.objects.get(name='dns_settings') static_hosts = StaticHost.objects.all() @@ -106,11 +56,21 @@ def generate_dnsdist_config(): return dnsdist_config -def generate_dnsmasq_config(): +def generate_dnsmasq_config(cluster_node=False): dns_settings = DNSSettings.objects.get(name='dns_settings') static_hosts = StaticHost.objects.all() dns_lists = DNSFilterList.objects.filter(enabled=True) - dnsmasq_config = f''' + if cluster_node: + # On cluster nodes dnsmasq shares the network namespace of the node and must also answer on + # WireGuard interfaces created after it starts + dnsmasq_config = ''' +no-dhcp-interface= +interface=wg* +bind-dynamic + +''' + else: + dnsmasq_config = ''' no-dhcp-interface= listen-address=0.0.0.0 bind-interfaces diff --git a/dns/views.py b/dns/views.py index 092628a..5286b6a 100644 --- a/dns/views.py +++ b/dns/views.py @@ -13,7 +13,7 @@ from django.utils.translation import gettext_lazy as _ from user_manager.models import UserAcl from .forms import DNSFilterListForm from .forms import DNSSettingsForm, StaticHostForm -from .functions import generate_dnsmasq_config, compress_dnsmasq_config +from .functions import generate_dnsmasq_config from .models import DNSFilterList, DNSSettings from .models import StaticHost @@ -44,7 +44,6 @@ def export_dns_configuration(): dnsmasq_config = generate_dnsmasq_config() with open(settings.DNS_CONFIG_FILE, 'w') as f: f.write(dnsmasq_config) - compress_dnsmasq_config() dns_settings, dns_settings_created = DNSSettings.objects.get_or_create(name='dns_settings') dns_settings.pending_changes = False dns_settings.save(update_fields=['pending_changes', 'updated']) diff --git a/docker-compose-caddy.yml b/docker-compose-caddy.yml index 3921f0f..de1509e 100644 --- a/docker-compose-caddy.yml +++ b/docker-compose-caddy.yml @@ -24,6 +24,7 @@ services: - app_secrets:/app_secrets/ - rrd_data:/rrd_data/ - caddy_json_export:/caddy_json_export/ + - caddy_cert_export:/caddy_cert_export/:ro ports: # Do not directly expose the Django port to the internet, use the reverse proxy below instead # - "127.0.0.1:8000:8000" @@ -87,6 +88,7 @@ services: volumes: - static_volume:/static - caddy_json_export:/caddy_json_export + - caddy_cert_export:/caddy_cert_export - caddy_data:/data - caddy_config:/config network_mode: "service:wireguard-webadmin" @@ -115,6 +117,7 @@ volumes: app_secrets: rrd_data: caddy_json_export: + caddy_cert_export: auth_gateway_data: caddy_data: caddy_config: diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 622d3e6..acc05e0 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -27,6 +27,7 @@ services: - app_secrets:/app_secrets/ - rrd_data:/rrd_data/ - caddy_json_export:/caddy_json_export/ + - caddy_cert_export:/caddy_cert_export/:ro ports: # Do not directly expose the Django port to the internet, use some kind of reverse proxy with SSL. - "8000:8000" @@ -99,6 +100,7 @@ services: volumes: - static_volume:/static - caddy_json_export:/caddy_json_export + - caddy_cert_export:/caddy_cert_export - caddy_data:/data - caddy_config:/config network_mode: "service:wireguard-webadmin" @@ -130,6 +132,7 @@ volumes: app_secrets: rrd_data: caddy_json_export: + caddy_cert_export: auth_gateway_data: caddy_data: caddy_config: diff --git a/firewall/tools.py b/firewall/tools.py index 48eff66..dec4591 100644 --- a/firewall/tools.py +++ b/firewall/tools.py @@ -6,6 +6,9 @@ from django.utils.translation import gettext_lazy as _ from firewall.models import FirewallRule, FirewallSettings, RedirectRule from wireguard.models import PeerAllowedIP, WireGuardInstance, Peer +PRIMARY_HOSTNAME = 'wireguard-webadmin' +CLUSTER_NODE_HOSTNAME = 'cluster-node' + def get_peer_addresses(peers, include_networks): addresses = [] @@ -213,19 +216,21 @@ def generate_route_policy_rules(): return route_policy_rules -def generate_firewall_header(): - firewall_settings, firewall_settings_created = FirewallSettings.objects.get_or_create(name='global') - header = f'''#!/bin/bash +def generate_firewall_header(cluster_node=False): + header = '''#!/bin/bash # Description: Firewall rules for WireGuard_WebAdmin # Do not edit this file directly. Use the web interface to manage firewall rules. # -# This script was generated by WireGuard_WebAdmin on {timezone.now().strftime('%Y-%m-%d %H:%M:%S %Z')} -# -DNS_IP=$(getent hosts wireguard-webadmin-dns | awk '{{ print $1 }}') +''' + if not cluster_node: + # Cluster nodes compare scripts by hash, so their copy must not change on every generation + header += f"# This script was generated by WireGuard_WebAdmin on {timezone.now().strftime('%Y-%m-%d %H:%M:%S %Z')}\n#\n" + header += '''DNS_IP=$(getent hosts wireguard-webadmin-dns | awk '{ print $1 }') if [ -z "$DNS_IP" ]; then DNS_IP="127.0.0.250" fi - +''' + header += ''' iptables -t nat -N WGWADM_POSTROUTING >> /dev/null 2>&1 iptables -t nat -N WGWADM_PREROUTING >> /dev/null 2>&1 iptables -t filter -N WGWADM_FORWARD >> /dev/null 2>&1 @@ -249,7 +254,7 @@ iptables -t filter -A FORWARD -m state --state RELATED,ESTABLISHED -j ACCEPT iptables -t filter -A FORWARD -i wg+ -j WGWADM_ROUTE_POLICY iptables -t filter -A FORWARD -j WGWADM_FORWARD ''' - if not settings.VPN_CLIENTS_CAN_ACCESS_DJANGO: + if not cluster_node and not settings.VPN_CLIENTS_CAN_ACCESS_DJANGO: header += 'iptables -t filter -A INPUT -i wg+ -p tcp --dport 8000 -j REJECT\n' return header @@ -276,7 +281,7 @@ def generate_firewall_footer(): return footer -def generate_port_forward_firewall(): +def generate_port_forward_firewall(server_hostname=PRIMARY_HOSTNAME): firewall_settings, firewall_settings_created = FirewallSettings.objects.get_or_create(name='global') redirect_firewall = '' wan_interface = firewall_settings.wan_interface @@ -298,7 +303,7 @@ def generate_port_forward_firewall(): rule_destination = peer_allowed_ip_address.allowed_ip if rule_destination: rule_text = f"# {redirect_rule.port}/{redirect_rule.protocol} - {redirect_rule.uuid} - Port Forward Rule set{description}\n" - rule_text += f"iptables -t nat -A WGWADM_PREROUTING -p {redirect_rule.protocol} -d wireguard-webadmin -i {wan_interface} --dport {redirect_rule.port} -j DNAT --to-dest {rule_destination}:{destination_port}\n" + rule_text += f"iptables -t nat -A WGWADM_PREROUTING -p {redirect_rule.protocol} -d {server_hostname} -i {wan_interface} --dport {redirect_rule.port} -j DNAT --to-dest {rule_destination}:{destination_port}\n" if redirect_rule.masquerade_source: rule_text += f"iptables -t nat -A WGWADM_POSTROUTING -p {redirect_rule.protocol} -d {rule_destination} -o wg+ --dport {destination_port} -j MASQUERADE\n" @@ -312,4 +317,20 @@ def generate_port_forward_firewall(): rule_text = f"# {redirect_rule.port}/{redirect_rule.protocol} - {redirect_rule.uuid} - Port Forward Rule set{description} - Missing IP for selected peer: {redirect_rule.peer}\n" redirect_firewall += rule_text - return redirect_firewall \ No newline at end of file + return redirect_firewall + +def generate_firewall_script(cluster_node=False): + """ + Build wg-firewall.sh for the primary server or for a cluster node. On cluster nodes dnsmasq shares the + network namespace of the node, so DNS is answered directly on the instance address without DNAT. + """ + firewall_content = generate_firewall_header(cluster_node=cluster_node) + if not cluster_node: + firewall_content += generate_redirect_dns_rules() + firewall_content += generate_port_forward_firewall( + server_hostname=CLUSTER_NODE_HOSTNAME if cluster_node else PRIMARY_HOSTNAME + ) + firewall_content += generate_route_policy_rules() + firewall_content += export_user_firewall() + firewall_content += generate_firewall_footer() + return firewall_content diff --git a/gatekeeper/views.py b/gatekeeper/views.py index 2ec8d68..dcd0c26 100644 --- a/gatekeeper/views.py +++ b/gatekeeper/views.py @@ -10,7 +10,7 @@ from django.shortcuts import render, get_object_or_404, redirect from django.urls import reverse from django.utils.translation import gettext as _ -from app_gateway.caddy_config_export import caddy_export_pending +from app_gateway.caddy_config_export import auto_apply_caddy_config, caddy_export_pending from app_gateway.setup_defaults import create_default_entries from gatekeeper.forms import GatekeeperUserForm, GatekeeperGroupForm, AuthMethodForm, AuthMethodAllowedDomainForm, \ AuthMethodAllowedEmailForm, GatekeeperIPAddressForm @@ -68,6 +68,7 @@ def view_manage_gatekeeper_user(request): if form.is_valid(): form.save() messages.success(request, _('Gatekeeper User saved successfully.')) + auto_apply_caddy_config(request) return redirect(cancel_url) form_description = { @@ -110,6 +111,7 @@ def view_delete_gatekeeper_user(request): if request.method == 'POST': gatekeeper_user.delete() messages.success(request, _('Gatekeeper User deleted successfully.')) + auto_apply_caddy_config(request) return redirect(cancel_url) context = { @@ -141,6 +143,7 @@ def view_manage_gatekeeper_group(request): if form.is_valid(): form.save() messages.success(request, _('Gatekeeper Group saved successfully.')) + auto_apply_caddy_config(request) return redirect(cancel_url) context = { @@ -163,6 +166,7 @@ def view_delete_gatekeeper_group(request): if request.method == 'POST': gatekeeper_group.delete() messages.success(request, _('Gatekeeper Group deleted successfully.')) + auto_apply_caddy_config(request) return redirect(cancel_url) context = { @@ -194,6 +198,7 @@ def view_manage_auth_method(request): if form.is_valid(): form.save() messages.success(request, _('Authentication Method saved successfully.')) + auto_apply_caddy_config(request) return redirect(cancel_url) form_description = { @@ -243,6 +248,7 @@ def view_delete_auth_method(request): if request.method == 'POST': auth_method.delete() messages.success(request, _('Authentication Method deleted successfully.')) + auto_apply_caddy_config(request) return redirect(cancel_url) context = { @@ -313,6 +319,7 @@ def view_manage_auth_domain(request): if form.is_valid(): form.save() messages.success(request, _('Allowed Domain saved successfully.')) + auto_apply_caddy_config(request) return redirect(cancel_url) context = { @@ -335,6 +342,7 @@ def view_delete_auth_domain(request): if request.method == 'POST': auth_domain.delete() messages.success(request, _('Allowed Domain deleted successfully.')) + auto_apply_caddy_config(request) return redirect(cancel_url) context = { @@ -366,6 +374,7 @@ def view_manage_auth_email(request): if form.is_valid(): form.save() messages.success(request, _('Allowed Email saved successfully.')) + auto_apply_caddy_config(request) return redirect(cancel_url) context = { @@ -388,6 +397,7 @@ def view_delete_auth_email(request): if request.method == 'POST': auth_email.delete() messages.success(request, _('Allowed Email deleted successfully.')) + auto_apply_caddy_config(request) return redirect(cancel_url) context = { @@ -419,6 +429,7 @@ def view_manage_gatekeeper_ip(request): if form.is_valid(): form.save() messages.success(request, _('IP Address saved successfully.')) + auto_apply_caddy_config(request) return redirect(cancel_url) form_description = { @@ -460,6 +471,7 @@ def view_delete_gatekeeper_ip(request): if request.method == 'POST': gatekeeper_ip.delete() messages.success(request, _('IP Address deleted successfully.')) + auto_apply_caddy_config(request) return redirect(cancel_url) context = { diff --git a/init.sh b/init.sh index a41e56f..7faa9e6 100644 --- a/init.sh +++ b/init.sh @@ -1,6 +1,11 @@ #!/bin/bash set -e +# Containers sharing this network namespace (Caddy) restart themselves when it changes +if [ -d /caddy_json_export ]; then + readlink /proc/self/ns/net > /caddy_json_export/.netns_owner +fi + # Lets wait for the DNS container to start sleep 5 diff --git a/templates/app_gateway/application_details.html b/templates/app_gateway/application_details.html index 5add9fb..caabbb0 100644 --- a/templates/app_gateway/application_details.html +++ b/templates/app_gateway/application_details.html @@ -195,17 +195,18 @@ $.getJSON("{% url 'application_certificate_status' %}?uuid={{ application.uuid }}", function (data) { $('.certificate-status').each(function () { var cell = $(this); - var hostStatus = data.hosts[cell.data('host-uuid')]; + var serversStatus = data.hosts[cell.data('host-uuid')] || []; cell.empty(); - if (!hostStatus) { - return; - } - cell.append( - $('') - .addClass('badge-' + hostStatus.css_class) - .attr('title', hostStatus.detail) - .text(hostStatus.label) - ); + $.each(serversStatus, function (index, serverStatus) { + var title = serverStatus.server ? serverStatus.server + ': ' + serverStatus.detail : serverStatus.detail; + var label = serverStatus.server ? serverStatus.server + ': ' + serverStatus.label : serverStatus.label; + cell.append( + $('') + .addClass('badge-' + serverStatus.css_class) + .attr('title', title) + .text(label) + ); + }); }); }).fail(function () { $('.certificate-status').html('-'); diff --git a/templates/cluster/worker_details.html b/templates/cluster/worker_details.html new file mode 100644 index 0000000..b68b294 --- /dev/null +++ b/templates/cluster/worker_details.html @@ -0,0 +1,168 @@ +{% extends "base.html" %} +{% load i18n %} + +{% block content %} +| {% trans 'Section' %} | {% trans 'Status' %} | {% trans 'Last update' %} |
|---|---|---|
{{ section.name }} |
+
+ {% if section.state == 'synced' %}{% trans 'Synchronized' %}
+ {% elif section.state == 'error' %}{% trans 'Error' %}
+ {% else %}{% trans 'Pending' %}{% endif %}
+ {% if section.message %} {{ section.message|truncatechars:300 }}{% endif %} + |
+ {{ section.updated|date:"SHORT_DATETIME_FORMAT"|default:'-' }} | +
| {{ certificate.hostname }} | ++ {% if certificate.status == 'trusted' %}{% trans 'Trusted' %} + {% elif certificate.status == 'untrusted' %}{% trans 'Self-signed' %} + {% else %}{% trans 'Unavailable' %}{% endif %} + {{ certificate.issuer }}{% if certificate.not_after %} · {{ certificate.not_after|date:"SHORT_DATE_FORMAT" }}{% endif %} + | +
{% blocktrans trimmed %}
+ Copy cluster_node.yml (and cluster_node-gateway.yml to publish App Gateway applications)
+ from the project repository
+ to the worker, and create a .env file next to them:
+ {% endblocktrans %}
MASTER_SERVER_ADDRESS={{ primary_address }}
+TOKEN={{ worker.token }}
+TIMEZONE={{ time_zone }}
+ {% trans 'VPN only' %}
+docker compose -f cluster_node.yml up -d+
{% trans 'VPN and App Gateway' %}
+docker compose -f cluster_node.yml -f cluster_node-gateway.yml up -d+