Rework cluster into v2: section-based worker sync and App Gateway on workers

This commit is contained in:
Eduardo Silva
2026-11-20 11:56:30 -03:00
parent dee270695d
commit 7468cdf623
41 changed files with 2766 additions and 1014 deletions
+155 -27
View File
@@ -1,11 +1,16 @@
import hashlib
import json
import os
import shutil
from django.conf import settings
from django.contrib import messages
from django.utils.translation import gettext as _
from app_gateway.models import (
AccessPolicy, Application, ApplicationHost, ApplicationPolicy, RESERVED_APP_NAME
)
from cluster.models import Worker
from gatekeeper.models import (
AuthMethod, GatekeeperGroup, GatekeeperIPAddress, GatekeeperUser
)
@@ -16,11 +21,43 @@ POLICY_TYPE_MAP = {
'deny': 'deny',
}
# Written by the Caddy container of the primary server, read-only for Django
CERTIFICATE_EXPORT_DIR = '/caddy_cert_export'
ACME_CERTIFICATE_TYPES = ('http_challenge', 'dns_challenge')
def build_applications_data():
applications = Application.objects.exclude(name=RESERVED_APP_NAME).prefetch_related('hosts')
def get_served_applications(worker=None):
"""Applications published by the primary server (worker=None) or by a cluster worker."""
applications = Application.objects.prefetch_related('hosts', 'workers')
if worker is not None:
applications = applications.exclude(name=RESERVED_APP_NAME)
return [application for application in applications if application.is_served_by(worker)]
def certificate_file_name(certificate):
return hashlib.sha256(certificate.encode()).hexdigest()
def read_exported_certificate(hostname):
"""Return (certificate, private_key) exported for a hostname, or (None, None)."""
if '/' in hostname or hostname in ('', '.', '..'):
return None, None
certificate_dir = os.path.join(CERTIFICATE_EXPORT_DIR, hostname)
try:
with open(os.path.join(certificate_dir, 'certificate.crt'), 'r', encoding='utf-8') as certificate_file:
certificate = certificate_file.read()
with open(os.path.join(certificate_dir, 'certificate.key'), 'r', encoding='utf-8') as key_file:
private_key = key_file.read()
except OSError:
return None, None
return certificate, private_key
def build_applications_data(worker=None):
entries = []
for app in applications:
for app in get_served_applications(worker):
if app.name == RESERVED_APP_NAME:
continue
entry = {
'id': app.name,
'name': app.display_name or app.name,
@@ -67,7 +104,8 @@ def _build_auth_method_entry(method):
return entry
def build_auth_policies_data():
def build_auth_policies_data(worker=None):
# Policies, methods, groups and users are referenced by name, so every server receives all of them
auth_methods = {}
for method in AuthMethod.objects.all():
auth_methods[method.name] = _build_auth_method_entry(method)
@@ -102,13 +140,9 @@ def build_auth_policies_data():
}
def build_routes_data():
applications = (
Application.objects
.prefetch_related('routes__policy')
)
def build_routes_data(worker=None):
entries = {}
for app in applications:
for app in get_served_applications(worker):
try:
app_policy = ApplicationPolicy.objects.get(application=app)
default_policy = app_policy.default_policy.name
@@ -132,15 +166,41 @@ def build_routes_data():
return {'entries': entries}
def build_tls_data():
def build_tls_data(worker=None):
"""
On the primary server every ACME certificate is issued, including the ones of applications that are only
served by cluster workers (certificate_only), and certificates used by workers are exported. Workers never
issue ACME certificates: they receive the exported certificate (certificate_file) or use the internal CA.
"""
hosts = {}
dns_credentials = {}
for application_host in ApplicationHost.objects.select_related('dns_credential'):
served_application_ids = {application.pk for application in get_served_applications(worker)}
application_hosts = ApplicationHost.objects.select_related('dns_credential', 'application').prefetch_related(
'application__workers'
)
for application_host in application_hosts:
application = application_host.application
hostname = application_host.hostname.lower()
is_acme = application_host.certificate_type in ACME_CERTIFICATE_TYPES
host_entry = {
'certificate_type': application_host.certificate_type,
'certificate_fallback': application_host.certificate_fallback,
'certificate_generation': application_host.certificate_generation,
}
if worker is not None:
if application.pk in served_application_ids:
hosts[hostname] = host_entry
continue
served_by_workers = application.name != RESERVED_APP_NAME and application.served_by_workers
if application.pk not in served_application_ids and not (served_by_workers and is_acme):
continue
host_entry['certificate_generation'] = application_host.certificate_generation
if served_by_workers and is_acme:
host_entry['export_certificate'] = True
if application.pk not in served_application_ids:
host_entry['certificate_only'] = True
dns_credential = application_host.dns_credential
if application_host.certificate_type == 'dns_challenge' and dns_credential:
host_entry['dns_credential'] = dns_credential.name
@@ -148,7 +208,7 @@ def build_tls_data():
'provider': dns_credential.provider,
'credentials': dns_credential.credentials,
}
hosts[application_host.hostname.lower()] = host_entry
hosts[hostname] = host_entry
return {
'hosts': hosts,
@@ -166,30 +226,98 @@ EXPORT_FILES = {
}
def export_caddy_config(output_dir):
os.makedirs(output_dir, exist_ok=True)
# Cluster workers follow the configuration exported for them, so changes are only published by an export
WORKER_EXPORT_DIR_NAME = 'cluster'
def get_worker_export_dir(output_dir, worker):
return os.path.join(output_dir, WORKER_EXPORT_DIR_NAME, str(worker.uuid))
def _write_export_files(target_dir, worker=None):
os.makedirs(target_dir, exist_ok=True)
for filename, builder in EXPORT_FILES.items():
filepath = os.path.join(output_dir, filename)
filepath = os.path.join(target_dir, filename)
with open(filepath, 'w', encoding='utf-8') as output_file:
json.dump(builder(), output_file, indent=2)
json.dump(builder(worker), output_file, indent=2)
output_file.write('\n')
def has_pending_export(output_dir):
"""Return True when the current configuration differs from the files last exported to output_dir."""
for filename, builder in EXPORT_FILES.items():
filepath = os.path.join(output_dir, filename)
def export_caddy_config(output_dir):
os.makedirs(output_dir, exist_ok=True)
workers = list(Worker.objects.all())
for worker in workers:
_write_export_files(get_worker_export_dir(output_dir, worker), worker)
workers_export_dir = os.path.join(output_dir, WORKER_EXPORT_DIR_NAME)
if os.path.isdir(workers_export_dir):
worker_uuids = {str(worker.uuid) for worker in workers}
for worker_dir_name in os.listdir(workers_export_dir):
if worker_dir_name not in worker_uuids:
shutil.rmtree(os.path.join(workers_export_dir, worker_dir_name), ignore_errors=True)
# The primary server is written last: its files are the ones Caddy and the auth-gateway watch
_write_export_files(output_dir)
def load_exported_files(target_dir):
"""Return {filename: data} from an export directory, or None when it is missing or incomplete."""
exported_files = {}
for filename in EXPORT_FILES:
try:
with open(filepath, 'r', encoding='utf-8') as exported_file:
exported_data = json.load(exported_file)
with open(os.path.join(target_dir, filename), 'r', encoding='utf-8') as exported_file:
exported_files[filename] = json.load(exported_file)
except (OSError, ValueError):
return True
# Round-trip through JSON so the comparison matches what export_caddy_config writes
if json.loads(json.dumps(builder())) != exported_data:
return None
return exported_files
def load_worker_export(worker, output_dir=None):
return load_exported_files(get_worker_export_dir(output_dir or CADDY_EXPORT_DIR, worker))
def add_certificate_files(tls_data):
"""Point the ACME hosts of a worker to the certificates currently exported by the primary server."""
for hostname, host_tls in tls_data.get('hosts', {}).items():
if host_tls.get('certificate_type') not in ACME_CERTIFICATE_TYPES:
continue
certificate, private_key = read_exported_certificate(hostname)
host_tls['certificate_file'] = certificate_file_name(certificate) if certificate and private_key else ''
return tls_data
def _export_differs(target_dir, worker=None):
exported_files = load_exported_files(target_dir)
if exported_files is None:
return True
# Round-trip through JSON so the comparison matches what export_caddy_config writes
for filename, builder in EXPORT_FILES.items():
if json.loads(json.dumps(builder(worker))) != exported_files[filename]:
return True
return False
def has_pending_export(output_dir):
"""Return True when the current configuration differs from the files last exported to output_dir."""
if _export_differs(output_dir):
return True
return any(_export_differs(get_worker_export_dir(output_dir, worker), worker) for worker in Worker.objects.all())
def caddy_export_pending():
return settings.CADDY_ENABLED and has_pending_export(CADDY_EXPORT_DIR)
def apply_caddy_config():
"""Export the configuration read by Caddy, the auth-gateway and the cluster workers."""
export_caddy_config(CADDY_EXPORT_DIR)
if settings.DEBUG:
export_caddy_config(os.path.join(settings.BASE_DIR, 'containers', 'caddy', 'config_files'))
def auto_apply_caddy_config(request):
if not settings.AUTO_APPLY or not settings.CADDY_ENABLED:
return False
apply_caddy_config()
messages.info(request, _('Gateway configuration applied automatically.'))
return True
+47 -1
View File
@@ -12,22 +12,44 @@ from app_gateway.models import (
Application, ApplicationHost, AccessPolicy, ApplicationPolicy, ApplicationRoute, DnsChallengeCredential
)
from app_gateway.setup_defaults import is_ip_address
from cluster.models import ClusterSettings, Worker
class ApplicationForm(forms.ModelForm):
class Meta:
model = Application
fields = ['display_name', 'upstream', 'allow_invalid_cert']
fields = ['display_name', 'upstream', 'allow_invalid_cert', 'serve_on_primary', 'all_workers', 'workers']
labels = {
'display_name': _('Name'),
'upstream': _('Upstream'),
'allow_invalid_cert': _('Allow invalid/self-signed certificate'),
'serve_on_primary': _('Serve on the primary server'),
'all_workers': _('Serve on all cluster workers'),
'workers': _('Cluster workers'),
}
def __init__(self, *args, **kwargs):
cancel_url = kwargs.pop('cancel_url', '#')
super().__init__(*args, **kwargs)
self.fields['display_name'].required = True
self.cluster_enabled = ClusterSettings.cluster_enabled()
cluster_rows = []
if self.cluster_enabled:
self.fields['workers'].queryset = Worker.objects.order_by('name')
self.fields['workers'].help_text = _('Select specific workers, or leave empty when serving on all cluster workers.')
cluster_rows = [
HTML(f'<hr><h5>{_("Cluster")}</h5><p class="text-muted">{_("Servers that publish this application.")}</p>'),
Div(
Div('serve_on_primary', css_class='col-md-12'),
Div('all_workers', css_class='col-md-12'),
Div('workers', css_class='col-md-12'),
css_class='row'
),
]
else:
for field_name in ('serve_on_primary', 'all_workers', 'workers'):
del self.fields[field_name]
self.helper = FormHelper()
self.helper.layout = Layout(
@@ -40,6 +62,7 @@ class ApplicationForm(forms.ModelForm):
Div('allow_invalid_cert', css_class='col-md-12'),
css_class='row'
),
*cluster_rows,
Div(
Div(
Submit('submit', _('Save'), css_class='btn btn-primary'),
@@ -71,6 +94,22 @@ class ApplicationForm(forms.ModelForm):
elif parsed.scheme == "http" and cleaned_data.get("allow_invalid_cert"):
self.add_error("allow_invalid_cert", _("This option is only available for https:// upstreams."))
if self.cluster_enabled:
all_workers = cleaned_data.get('all_workers')
selected_workers = cleaned_data.get('workers')
if all_workers and selected_workers:
self.add_error('workers', _('Leave this empty when the application is served on all cluster workers.'))
if not cleaned_data.get('serve_on_primary') and not all_workers and not selected_workers:
self.add_error(None, _('Select at least one server to publish this application.'))
served_by_workers = all_workers or bool(selected_workers)
if served_by_workers and self.instance.pk:
http_hosts = self.instance.hosts.filter(certificate_type='http_challenge').values_list('hostname', flat=True)
if http_hosts:
self.add_error(None, _(
'Applications served by cluster workers require DNS validation or self-signed certificates. '
'Change the certificate of these hosts first: %(hosts)s'
) % {'hosts': ', '.join(http_hosts)})
return cleaned_data
@@ -99,6 +138,12 @@ class ApplicationHostForm(forms.ModelForm):
if hostname.lower() in webadmin_hostnames:
self.add_error('hostname', _('This hostname is already used by WireGuard WebAdmin.'))
application = self.instance.application if self.instance.pk else self.application
if certificate_type == 'http_challenge' and application and application.served_by_workers:
self.add_error('certificate_type', _(
'This application is served by cluster workers. Use DNS validation or a self-signed certificate.'
))
if hostname and is_ip_address(hostname) and certificate_type != 'self_signed':
self.add_error('certificate_type', _('IP addresses can only use a self-signed certificate.'))
@@ -117,6 +162,7 @@ class ApplicationHostForm(forms.ModelForm):
def __init__(self, *args, **kwargs):
cancel_url = kwargs.pop('cancel_url', '#')
self.is_reserved = kwargs.pop('is_reserved', False)
self.application = kwargs.pop('application', None)
super().__init__(*args, **kwargs)
self.fields['dns_credential'].required = False
if self.is_reserved:
@@ -1,11 +1,10 @@
import os
import sys
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from django.db import transaction
from app_gateway.caddy_config_export import export_caddy_config
from app_gateway.caddy_config_export import apply_caddy_config
from app_gateway.models import AccessPolicy, Application, ApplicationPolicy, RESERVED_APP_NAME
from app_gateway.setup_defaults import create_default_entries
@@ -86,9 +85,7 @@ class Command(BaseCommand):
if settings.CADDY_ENABLED:
self.stdout.write('Exporting Caddy configuration...')
export_caddy_config('/caddy_json_export/')
if settings.DEBUG:
export_caddy_config(os.path.join(settings.BASE_DIR, 'containers', 'caddy', 'config_files'))
apply_caddy_config()
self.stdout.write(self.style.SUCCESS('Caddy configuration exported.'))
else:
self.stdout.write('Skipping Caddy configuration export (CADDY_ENABLED is not set).')
@@ -0,0 +1,29 @@
# Generated by Django 5.2.19 on 2026-11-19 16:17
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('app_gateway', '0012_certificate_type_labels'),
('cluster', '0018_cluster_v2'),
]
operations = [
migrations.AddField(
model_name='application',
name='all_workers',
field=models.BooleanField(default=False),
),
migrations.AddField(
model_name='application',
name='serve_on_primary',
field=models.BooleanField(default=True),
),
migrations.AddField(
model_name='application',
name='workers',
field=models.ManyToManyField(blank=True, related_name='applications', to='cluster.worker'),
),
]
+16
View File
@@ -4,6 +4,7 @@ from django.db import models
from django.utils.translation import gettext_lazy as _
from app_gateway.dns_providers import DNS_PROVIDER_CHOICES
from cluster.models import ClusterSettings
from gatekeeper.models import GatekeeperGroup, AuthMethod, _unique_slug
RESERVED_APP_NAME = 'wireguard_webadmin'
@@ -44,6 +45,9 @@ class Application(models.Model):
display_name = models.CharField(max_length=128, blank=True)
upstream = models.CharField(max_length=255, help_text=_("Upstream address, e.g.: http://10.188.18.27:3000"))
allow_invalid_cert = models.BooleanField(default=False, help_text=_("Allow invalid or self-signed TLS certificates from the upstream"))
serve_on_primary = models.BooleanField(default=True)
all_workers = models.BooleanField(default=False)
workers = models.ManyToManyField('cluster.Worker', blank=True, related_name='applications')
created = models.DateTimeField(auto_now_add=True)
updated = models.DateTimeField(auto_now=True)
@@ -59,6 +63,18 @@ class Application(models.Model):
def __str__(self):
return self.display_name or self.name
# Without an active cluster every application is served by the primary server only
@property
def served_by_workers(self):
return ClusterSettings.cluster_enabled() and (self.all_workers or self.workers.exists())
def is_served_by(self, worker):
if not ClusterSettings.cluster_enabled():
return worker is None
if worker is None:
return self.serve_on_primary
return self.all_workers or self.workers.filter(pk=worker.pk).exists()
class Meta:
ordering = ['name']
+306 -2
View File
@@ -8,13 +8,20 @@ from pathlib import Path
from unittest.mock import patch
from django.conf import settings
from django.test import TestCase
from django.contrib.auth.models import User
from django.test import TestCase, override_settings
from app_gateway.caddy_config_export import export_caddy_config
from app_gateway.models import AccessPolicy, Application, ApplicationHost, ApplicationPolicy, ApplicationRoute
from app_gateway.forms import ApplicationForm, ApplicationHostForm
from app_gateway.models import (
AccessPolicy, Application, ApplicationHost, ApplicationPolicy, ApplicationRoute, DnsChallengeCredential
)
from app_gateway.setup_defaults import create_default_entries
from cluster.models import ClusterSettings, Worker
from gatekeeper.models import (
AuthMethod, AuthMethodAllowedDomain, AuthMethodAllowedEmail, GatekeeperGroup, GatekeeperIPAddress, GatekeeperUser
)
from user_manager.models import UserAcl
AUTH_GATEWAY_FIXTURES_DIR = Path(settings.BASE_DIR) / 'containers' / 'auth-gateway' / 'tests' / 'fixtures' / 'config_files'
UPDATE_FIXTURES_ENV = 'UPDATE_AUTH_GATEWAY_FIXTURES'
@@ -29,6 +36,11 @@ _WEBADMIN_EXPORT_SPEC = importlib.util.spec_from_file_location('caddy_export_wir
_WEBADMIN_EXPORT_MODULE = importlib.util.module_from_spec(_WEBADMIN_EXPORT_SPEC)
_WEBADMIN_EXPORT_SPEC.loader.exec_module(_WEBADMIN_EXPORT_MODULE)
_PROCESS_CONFIG_PATH = Path(settings.BASE_DIR) / 'containers' / 'caddy' / 'process_config.py'
_PROCESS_CONFIG_SPEC = importlib.util.spec_from_file_location('caddy_process_config', _PROCESS_CONFIG_PATH)
process_config = importlib.util.module_from_spec(_PROCESS_CONFIG_SPEC)
_PROCESS_CONFIG_SPEC.loader.exec_module(process_config)
def create_auth_gateway_scenario():
local_password = AuthMethod.objects.create(display_name='Local Password', auth_type='local_password')
@@ -106,10 +118,16 @@ def export_auth_gateway_fixtures(output_dir):
_WEBADMIN_EXPORT_MODULE._write_config(os.path.join(output_dir, 'wireguard_webadmin.json'), webadmin_config)
# Same hosts used for wireguard_webadmin.json, so the WireGuard WebAdmin hosts do not depend on the local .env
FIXTURE_ALLOWED_HOSTS = ['wireguard-webadmin', 'webadmin.example.test', 'webadmin2.example.test', '10.0.0.1']
@override_settings(ALLOWED_HOSTS=FIXTURE_ALLOWED_HOSTS)
class AuthGatewayFixturesTests(TestCase):
maxDiff = None
def setUp(self):
create_default_entries()
create_auth_gateway_scenario()
def test_auth_gateway_fixtures_match_exporter(self):
@@ -131,3 +149,289 @@ class AuthGatewayFixturesTests(TestCase):
f"{filename} is out of date. Regenerate it with: "
f"{UPDATE_FIXTURES_ENV}=1 python manage.py test app_gateway"
)
CLOUDFLARE_TOKEN = 'cfut_' + 'a' * 40
def caddy_app(app_id, hosts):
return {'id': app_id, 'hosts': hosts, 'upstream': 'http://10.0.0.10:8080'}
def site_block(caddyfile, host):
"""Return the lines of the site block that starts with the given host."""
lines = caddyfile.splitlines()
start = next(index for index, line in enumerate(lines) if line.startswith(host) and line.endswith('{'))
end = next(index for index in range(start + 1, len(lines)) if lines[index] == '}')
return lines[start:end + 1]
class CaddyfileTlsTests(TestCase):
tls_data = {
'hosts': {
'http.example.test': {'certificate_type': 'http_challenge'},
'strict.example.test': {'certificate_type': 'http_challenge', 'certificate_fallback': False},
'dns.example.test': {'certificate_type': 'dns_challenge', 'dns_credential': 'cloudflare'},
'internal.example.test': {'certificate_type': 'self_signed'},
'worker-only.example.test': {'certificate_type': 'dns_challenge', 'dns_credential': 'cloudflare',
'certificate_only': True, 'export_certificate': True},
},
'dns_credentials': {'cloudflare': {'provider': 'cloudflare', 'credentials': {'api_token': CLOUDFLARE_TOKEN}}},
}
def build(self, apps, tls_data=None):
return process_config.build_caddyfile(apps, None, None, tls_data or self.tls_data)
def test_hosts_are_grouped_by_certificate_settings(self):
caddyfile = self.build([caddy_app('app', [
'http.example.test', 'strict.example.test', 'dns.example.test', 'internal.example.test', 'unknown.example.test'
])])
self.assertEqual(
site_block(caddyfile, 'http.example.test')[:5],
['http.example.test, unknown.example.test {', ' tls {', ' issuer acme', ' issuer internal', ' }'],
)
self.assertEqual(site_block(caddyfile, 'strict.example.test')[:4], ['strict.example.test {', ' tls {', ' issuer acme', ' }'])
self.assertEqual(site_block(caddyfile, 'internal.example.test')[1], ' tls internal')
dns_block = '\n'.join(site_block(caddyfile, 'dns.example.test'))
self.assertIn(f'api_token "{CLOUDFLARE_TOKEN}"', dns_block)
self.assertIn('propagation_delay 30s', dns_block)
self.assertIn('propagation_timeout -1', dns_block)
self.assertIn('issuer internal', dns_block)
def test_dns_challenge_can_be_disabled_after_invalid_configuration(self):
with patch.object(process_config, 'DISABLE_DNS_CHALLENGE', True):
caddyfile = self.build([caddy_app('app', ['dns.example.test'])])
self.assertNotIn('dns cloudflare', caddyfile)
self.assertIn('issuer acme', '\n'.join(site_block(caddyfile, 'dns.example.test')))
def test_primary_keeps_certificates_of_worker_only_hosts(self):
caddyfile = self.build([caddy_app('app', ['http.example.test'])])
worker_only_block = site_block(caddyfile, 'worker-only.example.test')
self.assertIn(' respond 404', worker_only_block)
self.assertIn(' dns cloudflare {', worker_only_block)
def test_cluster_node_never_requests_acme_certificates(self):
node_tls = {'hosts': {
'ready.example.test': {'certificate_type': 'dns_challenge', 'certificate_file': 'a' * 64},
'waiting.example.test': {'certificate_type': 'dns_challenge', 'certificate_file': ''},
'strict.example.test': {'certificate_type': 'dns_challenge', 'certificate_fallback': False, 'certificate_file': ''},
'internal.example.test': {'certificate_type': 'self_signed'},
}, 'dns_credentials': {}}
apps = [caddy_app('app', list(node_tls['hosts']))]
with patch.object(process_config, 'CLUSTER_NODE', True), \
patch.object(process_config, 'NODE_CERTIFICATES_DIR', '/caddy_certificates'):
caddyfile = self.build(apps, node_tls)
self.assertNotIn('issuer acme', caddyfile)
self.assertNotIn('strict.example.test', caddyfile)
certificate_path = '/caddy_certificates/' + 'a' * 64
self.assertEqual(site_block(caddyfile, 'ready.example.test')[1], f' tls {certificate_path}.crt {certificate_path}.key')
self.assertEqual(site_block(caddyfile, 'waiting.example.test')[0], 'waiting.example.test, internal.example.test {')
self.assertEqual(site_block(caddyfile, 'waiting.example.test')[1], ' tls internal')
class CaddyTlsStateTests(TestCase):
"""Fallback certificates are removed and Caddy is restarted only when the certificate settings change."""
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.addCleanup(self.temp_dir.cleanup)
self.certificates_dir = os.path.join(self.temp_dir.name, 'certificates')
self.json_dir = os.path.join(self.temp_dir.name, 'json')
self.export_dir = os.path.join(self.temp_dir.name, 'export')
for directory in (self.certificates_dir, self.json_dir, self.export_dir):
os.makedirs(directory)
for attribute, value in (
('CERTIFICATES_DIR', self.certificates_dir),
('TLS_STATE_PATH', os.path.join(self.temp_dir.name, 'tls_state.json')),
('TLS_STATE_PENDING_PATH', os.path.join(self.temp_dir.name, 'tls_state_pending.json')),
('JSON_DIR', self.json_dir),
('CERTIFICATE_EXPORT_DIR', self.export_dir),
):
patcher = patch.object(process_config, attribute, value)
patcher.start()
self.addCleanup(patcher.stop)
def store_certificate(self, issuer_dir_name, host, content='certificate'):
host_dir = os.path.join(self.certificates_dir, issuer_dir_name, host)
os.makedirs(host_dir, exist_ok=True)
for extension in ('crt', 'key'):
with open(os.path.join(host_dir, f'{host}.{extension}'), 'w', encoding='utf-8') as certificate_file:
certificate_file.write(f'{content}-{extension}')
return host_dir
def apply(self, host_tls):
apps = [caddy_app('app', ['app.example.test'])]
tls_data = {'hosts': {'app.example.test': host_tls}, 'dns_credentials': {}}
process_config._write_json(process_config.TLS_STATE_PENDING_PATH, process_config.build_tls_state(apps, tls_data))
output = io.StringIO()
with contextlib.redirect_stdout(output), contextlib.redirect_stderr(io.StringIO()):
process_config.sync_tls_state()
return output.getvalue().strip() == 'restart'
def test_first_run_never_restarts(self):
fallback_dir = self.store_certificate('local', 'app.example.test')
self.assertFalse(self.apply({'certificate_type': 'http_challenge'}))
self.assertTrue(os.path.isdir(fallback_dir))
def test_retry_removes_fallback_certificate(self):
self.apply({'certificate_type': 'http_challenge'})
fallback_dir = self.store_certificate('local', 'app.example.test')
self.assertTrue(self.apply({'certificate_type': 'http_challenge', 'certificate_generation': 1}))
self.assertFalse(os.path.isdir(fallback_dir))
def test_retry_keeps_valid_acme_certificate_without_restart(self):
self.apply({'certificate_type': 'http_challenge'})
acme_dir = self.store_certificate('acme-v02.api.letsencrypt.org-directory', 'app.example.test')
self.assertFalse(self.apply({'certificate_type': 'http_challenge', 'certificate_generation': 1}))
self.assertTrue(os.path.isdir(acme_dir))
def test_switching_to_self_signed_restarts_when_acme_certificate_is_cached(self):
self.apply({'certificate_type': 'http_challenge'})
self.store_certificate('acme-v02.api.letsencrypt.org-directory', 'app.example.test')
self.assertTrue(self.apply({'certificate_type': 'self_signed'}))
def test_unchanged_settings_do_not_restart(self):
self.apply({'certificate_type': 'http_challenge'})
self.store_certificate('local', 'app.example.test')
self.assertFalse(self.apply({'certificate_type': 'http_challenge'}))
def test_readded_host_with_leftover_fallback_is_retried(self):
self.apply({'certificate_type': 'http_challenge'})
fallback_dir = self.store_certificate('local', 'other.example.test')
apps = [caddy_app('app', ['app.example.test', 'other.example.test'])]
tls_data = {'hosts': {'app.example.test': {'certificate_type': 'http_challenge'},
'other.example.test': {'certificate_type': 'dns_challenge', 'dns_credential': 'cloudflare'}},
'dns_credentials': {'cloudflare': {'provider': 'cloudflare', 'credentials': {'api_token': CLOUDFLARE_TOKEN}}}}
process_config._write_json(process_config.TLS_STATE_PENDING_PATH, process_config.build_tls_state(apps, tls_data))
output = io.StringIO()
with contextlib.redirect_stdout(output), contextlib.redirect_stderr(io.StringIO()):
process_config.sync_tls_state()
self.assertEqual(output.getvalue().strip(), 'restart')
self.assertFalse(os.path.isdir(fallback_dir))
def test_new_host_without_cached_certificate_does_not_restart(self):
self.apply({'certificate_type': 'http_challenge'})
apps = [caddy_app('app', ['app.example.test', 'new.example.test'])]
tls_data = {'hosts': {'app.example.test': {'certificate_type': 'http_challenge'},
'new.example.test': {'certificate_type': 'http_challenge'}}, 'dns_credentials': {}}
process_config._write_json(process_config.TLS_STATE_PENDING_PATH, process_config.build_tls_state(apps, tls_data))
output = io.StringIO()
with contextlib.redirect_stdout(output), contextlib.redirect_stderr(io.StringIO()):
process_config.sync_tls_state()
self.assertEqual(output.getvalue().strip(), '')
def test_cluster_node_restarts_when_certificate_changes(self):
with patch.object(process_config, 'CLUSTER_NODE', True):
self.apply({'certificate_type': 'dns_challenge', 'certificate_file': ''})
fallback_dir = self.store_certificate('local', 'app.example.test')
self.assertTrue(self.apply({'certificate_type': 'dns_challenge', 'certificate_file': 'a' * 64}))
self.assertFalse(os.path.isdir(fallback_dir))
self.assertTrue(self.apply({'certificate_type': 'dns_challenge', 'certificate_file': 'b' * 64}))
def test_only_acme_certificates_are_exported(self):
tls_data = {'hosts': {
'exported.example.test': {'certificate_type': 'dns_challenge', 'export_certificate': True},
'fallback.example.test': {'certificate_type': 'dns_challenge', 'export_certificate': True},
'local.example.test': {'certificate_type': 'dns_challenge'},
}}
with open(os.path.join(self.json_dir, 'tls.json'), 'w', encoding='utf-8') as tls_file:
json.dump(tls_data, tls_file)
self.store_certificate('acme-v02.api.letsencrypt.org-directory', 'exported.example.test', 'acme')
self.store_certificate('acme-v02.api.letsencrypt.org-directory', 'local.example.test', 'acme')
self.store_certificate('local', 'fallback.example.test', 'internal')
os.makedirs(os.path.join(self.export_dir, 'removed.example.test'))
with contextlib.redirect_stdout(io.StringIO()):
process_config.export_certificates()
self.assertEqual(sorted(os.listdir(self.export_dir)), ['exported.example.test'])
with open(os.path.join(self.export_dir, 'exported.example.test', 'certificate.crt'), encoding='utf-8') as certificate_file:
self.assertEqual(certificate_file.read(), 'acme-crt')
@override_settings(ALLOWED_HOSTS=FIXTURE_ALLOWED_HOSTS)
class ClusterApplicationFormTests(TestCase):
def setUp(self):
ClusterSettings.objects.update_or_create(name='cluster_settings', defaults={'enabled': True})
self.worker = Worker.objects.create(name='node-a')
self.application = Application.objects.create(display_name='Wiki', upstream='http://10.0.0.30:8080')
self.credential = DnsChallengeCredential.objects.create(
display_name='Cloudflare', provider='cloudflare', credentials={'api_token': CLOUDFLARE_TOKEN}
)
def test_application_with_http_validation_hosts_cannot_be_served_by_workers(self):
ApplicationHost.objects.create(application=self.application, hostname='wiki.example.test')
form = ApplicationForm(data={
'display_name': 'Wiki', 'upstream': 'http://10.0.0.30:8080', 'serve_on_primary': 'on',
'workers': [self.worker.pk],
}, instance=self.application)
self.assertFalse(form.is_valid())
self.assertIn('wiki.example.test', str(form.errors))
def test_all_workers_and_specific_workers_are_exclusive(self):
form = ApplicationForm(data={
'display_name': 'Wiki', 'upstream': 'http://10.0.0.30:8080', 'serve_on_primary': 'on',
'all_workers': 'on', 'workers': [self.worker.pk],
}, instance=self.application)
self.assertIn('workers', form.errors)
def test_application_must_be_published_somewhere(self):
form = ApplicationForm(data={'display_name': 'Wiki', 'upstream': 'http://10.0.0.30:8080'}, instance=self.application)
self.assertFalse(form.is_valid())
self.assertIn('__all__', form.errors)
def test_without_cluster_everything_is_served_by_the_primary(self):
self.application.serve_on_primary = False
self.application.save()
self.application.workers.add(self.worker)
ClusterSettings.objects.filter(name='cluster_settings').update(enabled=False)
self.assertTrue(self.application.is_served_by(None))
self.assertFalse(self.application.is_served_by(self.worker))
self.assertFalse(self.application.served_by_workers)
form = ApplicationForm(data={'display_name': 'Wiki', 'upstream': 'http://10.0.0.30:8080'}, instance=self.application)
self.assertTrue(form.is_valid(), form.errors)
self.assertNotIn('workers', form.fields)
def test_host_of_application_served_by_workers_requires_dns_or_self_signed(self):
self.application.workers.add(self.worker)
form = ApplicationHostForm(
data={'hostname': 'wiki.example.test', 'certificate_type': 'http_challenge', 'certificate_fallback': 'on'},
application=self.application,
)
self.assertIn('certificate_type', form.errors)
form = ApplicationHostForm(data={
'hostname': 'wiki.example.test', 'certificate_type': 'dns_challenge',
'dns_credential': self.credential.pk, 'certificate_fallback': 'on',
}, application=self.application)
self.assertTrue(form.is_valid(), form.errors)
@override_settings(ALLOWED_HOSTS=FIXTURE_ALLOWED_HOSTS + ['testserver'], CADDY_ENABLED=True, DEBUG=False)
class GatewayAutoApplyTests(TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.addCleanup(self.temp_dir.cleanup)
patcher = patch('app_gateway.caddy_config_export.CADDY_EXPORT_DIR', self.temp_dir.name)
patcher.start()
self.addCleanup(patcher.stop)
user = User.objects.create_user(username='gateway-admin', password='unused-test-password')
UserAcl.objects.create(user=user, user_level=50)
self.client.force_login(user)
def save_application(self):
return self.client.post('/app_gateway/application/manage/', {
'display_name': 'Auto Apply', 'upstream': 'http://10.0.0.40:8080',
})
@override_settings(AUTO_APPLY=True)
def test_saving_applies_configuration(self):
self.save_application()
with open(os.path.join(self.temp_dir.name, 'applications.json'), encoding='utf-8') as applications_file:
self.assertIn('auto-apply', [entry['id'] for entry in json.load(applications_file)['entries']])
@override_settings(AUTO_APPLY=False)
def test_disabled_auto_apply_keeps_changes_pending(self):
self.save_application()
self.assertFalse(os.path.exists(os.path.join(self.temp_dir.name, 'applications.json')))
+104 -33
View File
@@ -1,4 +1,4 @@
import os
from datetime import timedelta
from django.conf import settings
from django.contrib import messages
@@ -7,13 +7,18 @@ from django.db.models import F, ProtectedError
from django.http import JsonResponse
from django.shortcuts import render, get_object_or_404, redirect
from django.urls import reverse
from django.utils import timezone
from django.utils.dateparse import parse_datetime
from django.utils.formats import date_format
from django.utils.http import url_has_allowed_host_and_scheme
from django.utils.timezone import localtime
from django.utils.translation import gettext as _
from django.views.decorators.http import require_POST
from app_gateway.caddy_config_export import CADDY_EXPORT_DIR, caddy_export_pending, export_caddy_config
from app_gateway.caddy_config_export import (
ACME_CERTIFICATE_TYPES, CADDY_EXPORT_DIR, apply_caddy_config, auto_apply_caddy_config, caddy_export_pending,
load_exported_files, load_worker_export
)
from app_gateway.certificate_status import inspect_certificate
from app_gateway.forms import (
ApplicationForm, ApplicationHostForm, AccessPolicyForm,
@@ -24,6 +29,7 @@ from app_gateway.models import (
RESERVED_APP_NAME
)
from app_gateway.setup_defaults import create_default_entries
from cluster.models import ClusterSettings, Worker
from user_manager.models import UserAcl
@@ -79,6 +85,36 @@ def view_application_details(request):
return render(request, 'app_gateway/application_details.html', context)
def worker_runs_gateway(worker):
# Workers running the App Gateway report the Caddy service; VPN-only workers do not serve applications
worker_status = getattr(worker, 'workerstatus', None)
return bool(worker_status and 'caddy' in (worker_status.services_status or {}))
def describe_certificate_status(certificate, certificate_type):
"""Badge information for a certificate inspected on the primary server or reported by a cluster worker."""
expiration = ''
if certificate.get('not_after'):
not_after = certificate['not_after']
if isinstance(not_after, str):
not_after = parse_datetime(not_after)
if not_after:
expiration = _('expires %(date)s') % {'date': date_format(localtime(not_after), 'SHORT_DATETIME_FORMAT')}
if certificate.get('status') == 'trusted':
return 'success', _('Trusted'), f"{certificate.get('issuer', '')}, {expiration}"
if certificate.get('status') == 'untrusted' and certificate_type == 'self_signed':
return 'secondary', _('Self-signed'), expiration
if certificate.get('status') == 'untrusted':
detail = _('The trusted certificate could not be issued yet, %(expiration)s') % {'expiration': expiration}
return 'warning', _('Using fallback'), detail
if certificate.get('status') == 'unknown':
return 'light', _('Unknown'), _('No recent report from this server.')
if certificate.get('status') == 'pending_export':
return 'info', _('Pending export'), _('Export the configuration to publish this host on this server.')
return 'danger', _('Unavailable'), _('No certificate is being served for this hostname.')
@login_required
def view_application_certificate_status(request):
if not UserAcl.objects.filter(user=request.user).filter(user_level__gte=20).exists():
@@ -88,29 +124,53 @@ def view_application_certificate_status(request):
if not settings.CADDY_ENABLED:
return JsonResponse({'caddy_enabled': False, 'hosts': {}})
workers = []
if application.name != RESERVED_APP_NAME and ClusterSettings.objects.filter(name='cluster_settings', enabled=True).exists():
workers = [
worker for worker in Worker.objects.filter(enabled=True).select_related('workerstatus').order_by('name')
if application.is_served_by(worker) and worker_runs_gateway(worker)
]
report_limit = timezone.now() - timedelta(minutes=10)
# Servers only publish what was exported, so hosts missing from their exported files are pending
primary_export = load_exported_files(CADDY_EXPORT_DIR) or {}
primary_exported_hosts = primary_export.get('tls.json', {}).get('hosts', {})
worker_exported_hosts = {}
for worker in workers:
worker_export = load_worker_export(worker) or {}
worker_exported_hosts[worker.pk] = worker_export.get('tls.json', {}).get('hosts', {})
hosts_status = {}
for application_host in application.hosts.all():
certificate = inspect_certificate(application_host.hostname)
expiration = ''
if certificate['not_after']:
expiration = _('expires %(date)s') % {
'date': date_format(localtime(certificate['not_after']), 'SHORT_DATETIME_FORMAT')
}
hostname = application_host.hostname.lower()
servers = []
# The primary server also manages ACME certificates of applications only served by workers
primary_handles_host = application.is_served_by(None) or (
application_host.certificate_type in ACME_CERTIFICATE_TYPES and application.served_by_workers
)
if primary_handles_host:
if hostname in primary_exported_hosts:
certificate = inspect_certificate(application_host.hostname)
else:
certificate = {'status': 'pending_export'}
servers.append((_('Primary Server'), certificate))
if certificate['status'] == 'trusted':
css_class, label = 'success', _('Trusted')
detail = f"{certificate['issuer']}, {expiration}"
elif certificate['status'] == 'untrusted' and application_host.certificate_type == 'self_signed':
css_class, label = 'secondary', _('Self-signed')
detail = expiration
elif certificate['status'] == 'untrusted':
css_class, label = 'warning', _('Using fallback')
detail = _('The trusted certificate could not be issued yet, %(expiration)s') % {'expiration': expiration}
else:
css_class, label = 'danger', _('Unavailable')
detail = _('No certificate is being served for this hostname.')
for worker in workers:
worker_status = getattr(worker, 'workerstatus', None)
if hostname not in worker_exported_hosts[worker.pk]:
certificate = {'status': 'pending_export'}
elif worker_status and worker_status.certificate_status_updated and worker_status.certificate_status_updated > report_limit:
certificate = worker_status.certificate_status.get(hostname, {'status': 'unavailable'})
else:
certificate = {'status': 'unknown'}
servers.append((worker.display_name, certificate))
hosts_status[str(application_host.uuid)] = {'css_class': css_class, 'label': label, 'detail': detail}
hosts_status[str(application_host.uuid)] = []
for server_name, certificate in servers:
css_class, label, detail = describe_certificate_status(certificate, application_host.certificate_type)
hosts_status[str(application_host.uuid)].append({
'server': server_name if workers else '', 'css_class': css_class, 'label': label, 'detail': detail,
})
return JsonResponse({'caddy_enabled': True, 'hosts': hosts_status})
@@ -132,13 +192,17 @@ def view_manage_application(request):
application = None
title = _('Create Application')
cancel_url = reverse('app_gateway_list') + '?tab=applications'
if application:
cancel_url = reverse('view_application') + f'?uuid={application.uuid}'
else:
cancel_url = reverse('app_gateway_list') + '?tab=applications'
form = ApplicationForm(request.POST or None, instance=application, cancel_url=cancel_url)
if form.is_valid():
form.save()
application = form.save()
messages.success(request, _('Application saved successfully.'))
return redirect(cancel_url)
auto_apply_caddy_config(request)
return redirect(reverse('view_application') + f'?uuid={application.uuid}')
form_description = {
'size': 'col-lg-6',
@@ -178,6 +242,7 @@ def view_delete_application(request):
if request.method == 'POST':
application.delete()
messages.success(request, _('Application deleted successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
context = {
@@ -214,13 +279,15 @@ def view_manage_application_host(request):
return redirect(cancel_url)
form = ApplicationHostForm(
request.POST or None, instance=application_host, cancel_url=cancel_url, is_reserved=is_reserved
request.POST or None, instance=application_host, cancel_url=cancel_url, is_reserved=is_reserved,
application=application,
)
if form.is_valid():
host = form.save(commit=False)
host.application = application
host.save()
messages.success(request, _('Application Host saved successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
form_description_content = _('''
@@ -233,6 +300,7 @@ def view_manage_application_host(request):
</ul>
<p>When the fallback option is enabled and a trusted certificate cannot be issued, a self-signed certificate is used until the next successful attempt. Without the fallback, the connection fails until the certificate is issued.</p>
<p>Use <strong>Retry certificate</strong> in the host list to request a new attempt right away, for example after fixing the DNS or firewall.</p>
<p><strong>Important:</strong> the certificate is requested as soon as the configuration is applied. With HTTP validation, the hostname must already point to this server: failed attempts count towards the Let's Encrypt limit (5 per hour per hostname). If the DNS is not ready yet, use DNS validation.</p>
''')
if is_reserved:
form_description_content += _('''
@@ -268,6 +336,7 @@ def view_delete_application_host(request):
if request.method == 'POST':
application_host.delete()
messages.success(request, _('Application Host deleted successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
context = {
@@ -312,6 +381,7 @@ def view_manage_access_policy(request):
if form.is_valid():
form.save()
messages.success(request, _('Access Policy saved successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
if policy_type == 'public':
@@ -365,6 +435,7 @@ def view_delete_access_policy(request):
try:
access_policy.delete()
messages.success(request, _('Access Policy deleted successfully.'))
auto_apply_caddy_config(request)
except ProtectedError:
messages.error(request, _('Cannot delete this Access Policy because it is currently in use by an Application Route or Application Default Policy.'))
return redirect(cancel_url)
@@ -403,6 +474,7 @@ def view_manage_application_policy(request):
policy_config.application = application
policy_config.save()
messages.success(request, _('Application Default Policy saved successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
context = {
@@ -426,6 +498,7 @@ def view_delete_application_policy(request):
if request.method == 'POST':
application_policy.delete()
messages.success(request, _('Application Default Policy deleted successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
context = {
@@ -464,6 +537,7 @@ def view_manage_application_route(request):
route.application = application
route.save()
messages.success(request, _('Application Route saved successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
form_description = {
@@ -502,6 +576,7 @@ def view_delete_application_route(request):
if request.method == 'POST':
application_route.delete()
messages.success(request, _('Application Route deleted successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
context = {
@@ -534,9 +609,7 @@ def view_retry_application_host_certificate(request):
return redirect(redirect_url)
ApplicationHost.objects.filter(pk=application_host.pk).update(certificate_generation=F('certificate_generation') + 1)
export_caddy_config(CADDY_EXPORT_DIR)
if settings.DEBUG:
export_caddy_config(os.path.join(settings.BASE_DIR, 'containers', 'caddy', 'config_files'))
apply_caddy_config()
messages.success(request, _(
'A new certificate will be requested for %(hostname)s. The current configuration was exported.'
@@ -564,6 +637,7 @@ def view_manage_dns_credential(request):
if form.is_valid():
form.save()
messages.success(request, _('DNS Credential saved successfully.'))
auto_apply_caddy_config(request)
return redirect(cancel_url)
context = {
@@ -587,6 +661,7 @@ def view_delete_dns_credential(request):
try:
dns_credential.delete()
messages.success(request, _('DNS Credential deleted successfully.'))
auto_apply_caddy_config(request)
except ProtectedError:
messages.error(request, _('Cannot delete this DNS Credential because it is currently in use by an Application Host.'))
return redirect(cancel_url)
@@ -618,10 +693,6 @@ def view_export_caddy_config(request):
))
return redirect(redirect_url)
export_caddy_config(CADDY_EXPORT_DIR)
if settings.DEBUG:
export_caddy_config(os.path.join(settings.BASE_DIR, 'containers', 'caddy', 'config_files'))
apply_caddy_config()
messages.success(request, _('Configuration exported successfully.'))
return redirect(redirect_url)