mirror of
https://github.com/eduardogsilva/wireguard_webadmin.git
synced 2026-11-22 04:43:18 +01:00
Rework cluster into v2: section-based worker sync and App Gateway on workers
This commit is contained in:
+5
-5
@@ -5,18 +5,18 @@ from .models import ClusterSettings, Worker, WorkerStatus
|
||||
|
||||
@admin.register(ClusterSettings)
|
||||
class ClusterSettingsAdmin(admin.ModelAdmin):
|
||||
list_display = ('name', 'enabled', 'cluster_mode', 'restart_mode', 'config_version', 'created', 'updated')
|
||||
list_filter = ('enabled', 'cluster_mode', 'restart_mode', 'primary_enable_wireguard')
|
||||
list_display = ('name', 'enabled', 'cluster_mode', 'created', 'updated')
|
||||
list_filter = ('enabled', 'cluster_mode', 'primary_enable_wireguard')
|
||||
|
||||
|
||||
@admin.register(Worker)
|
||||
class WorkerAdmin(admin.ModelAdmin):
|
||||
list_display = ('name', 'enabled', 'ip_address', 'country', 'city', 'ip_lock', 'created', 'updated')
|
||||
list_filter = ('enabled', 'ip_lock', 'country', 'force_reload', 'force_restart')
|
||||
list_filter = ('enabled', 'ip_lock', 'country')
|
||||
|
||||
|
||||
@admin.register(WorkerStatus)
|
||||
class WorkerStatusAdmin(admin.ModelAdmin):
|
||||
list_display = ('worker', 'last_seen', 'config_version', 'last_reload', 'last_restart')
|
||||
list_filter = ('last_seen', 'last_reload', 'last_restart')
|
||||
list_display = ('worker', 'last_seen', 'worker_version', 'wireguard_status_updated')
|
||||
list_filter = ('last_seen',)
|
||||
|
||||
|
||||
+132
-182
@@ -1,220 +1,170 @@
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
|
||||
from django.conf import settings
|
||||
from django.http import JsonResponse, FileResponse
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.http import JsonResponse
|
||||
from django.utils import timezone
|
||||
from django.views.decorators.csrf import csrf_exempt
|
||||
from django.views.decorators.gzip import gzip_page
|
||||
from django.views.decorators.http import require_GET, require_POST
|
||||
|
||||
from .models import ClusterSettings, Worker, WorkerStatus
|
||||
from cluster.models import ClusterSettings, Worker, WorkerStatus
|
||||
from cluster.sync import SECTION_NAMES, build_manifest, build_section
|
||||
|
||||
REPORT_STATUS_FIELDS = ('status', 'message')
|
||||
|
||||
|
||||
def get_ip_address(request):
|
||||
ip_address = request.META.get('HTTP_X_FORWARDED_FOR')
|
||||
if ip_address:
|
||||
ip_address = ip_address.split(',')[0]
|
||||
ip_address = ip_address.split(',')[0].strip()
|
||||
else:
|
||||
ip_address = request.META.get('REMOTE_ADDR')
|
||||
return ip_address
|
||||
|
||||
|
||||
def get_cluster_settings():
|
||||
cluster_settings, created = ClusterSettings.objects.get_or_create(name='cluster_settings')
|
||||
return {
|
||||
'enabled': cluster_settings.enabled,
|
||||
'primary_enable_wireguard': cluster_settings.primary_enable_wireguard,
|
||||
'stats_sync_interval': settings.WIREGUARD_STATUS_CACHE_REFRESH_INTERVAL,
|
||||
'cluster_mode': cluster_settings.cluster_mode,
|
||||
'restart_mode': cluster_settings.restart_mode,
|
||||
'config_version': cluster_settings.config_version,
|
||||
'dns_version': cluster_settings.dns_version,
|
||||
}
|
||||
def _error_response(message, status):
|
||||
return JsonResponse({'status': 'error', 'message': message}, status=status)
|
||||
|
||||
|
||||
def get_worker(request):
|
||||
success = True
|
||||
ip_address = get_ip_address(request)
|
||||
token = request.GET.get('token', '')
|
||||
"""
|
||||
Authenticate the worker from the Authorization header and update its status.
|
||||
Returns (worker, None) on success or (None, JsonResponse) when the request must be rejected.
|
||||
"""
|
||||
authorization = request.headers.get('Authorization', '')
|
||||
if not authorization.startswith('Bearer '):
|
||||
return None, _error_response('Worker not found', 403)
|
||||
try:
|
||||
worker = Worker.objects.get(token=token)
|
||||
except:
|
||||
return None, False
|
||||
worker = Worker.objects.get(token=authorization[len('Bearer '):].strip())
|
||||
except (Worker.DoesNotExist, ValidationError, ValueError):
|
||||
return None, _error_response('Worker not found', 403)
|
||||
|
||||
worker_status, created = WorkerStatus.objects.get_or_create(worker=worker)
|
||||
worker_status, worker_status_created = WorkerStatus.objects.get_or_create(worker=worker)
|
||||
try:
|
||||
worker_config_version = int(request.GET.get('worker_config_version'))
|
||||
worker_version = int(request.GET.get('worker_version'))
|
||||
worker_dns_version = int(request.GET.get('worker_dns_version'))
|
||||
except:
|
||||
worker.error_status = 'missing_version'
|
||||
worker.save()
|
||||
return worker, False
|
||||
|
||||
if worker.error_status == 'missing_version':
|
||||
worker.error_status = ''
|
||||
worker.save()
|
||||
|
||||
if worker_version < settings.CLUSTER_WORKER_MINIMUM_VERSION:
|
||||
worker.error_status = 'update_required'
|
||||
worker.save()
|
||||
return worker, False
|
||||
if worker.error_status == 'update_required':
|
||||
worker.error_status = ''
|
||||
worker.save()
|
||||
|
||||
if worker_status.config_version != worker_config_version:
|
||||
worker_status.config_version = worker_config_version
|
||||
if worker_status.worker_version != worker_version:
|
||||
worker_status.worker_version = worker_version
|
||||
if worker_status.dns_version != worker_dns_version:
|
||||
worker_status.dns_version = worker_dns_version
|
||||
worker_version = int(request.headers.get('X-Worker-Version', '0'))
|
||||
except ValueError:
|
||||
worker_version = 0
|
||||
worker_status.worker_version = worker_version
|
||||
worker_status.last_seen = timezone.now()
|
||||
worker_status.save()
|
||||
|
||||
if not worker.ip_address:
|
||||
worker_status.save(update_fields=['worker_version', 'last_seen', 'updated'])
|
||||
|
||||
ip_address = get_ip_address(request)
|
||||
error_status = ''
|
||||
if worker_version < settings.CLUSTER_WORKER_MINIMUM_VERSION:
|
||||
error_status = 'update_required'
|
||||
elif worker.ip_lock and worker.ip_address and worker.ip_address != ip_address:
|
||||
error_status = 'ip_lock'
|
||||
elif not worker.enabled:
|
||||
error_status = 'worker_disabled'
|
||||
else:
|
||||
cluster_settings, cluster_settings_created = ClusterSettings.objects.get_or_create(name='cluster_settings')
|
||||
if cluster_settings.enabled and not settings.WIREGUARD_STATUS_CACHE_ENABLED:
|
||||
cluster_settings.enabled = False
|
||||
cluster_settings.save()
|
||||
if not cluster_settings.enabled:
|
||||
error_status = 'cluster_disabled'
|
||||
|
||||
update_fields = []
|
||||
if not worker.ip_lock and worker.ip_address != ip_address:
|
||||
worker.ip_address = ip_address
|
||||
worker.save()
|
||||
update_fields.append('ip_address')
|
||||
if worker.error_status != error_status:
|
||||
worker.error_status = error_status
|
||||
update_fields.append('error_status')
|
||||
if update_fields:
|
||||
worker.save(update_fields=update_fields + ['updated'])
|
||||
|
||||
if worker.ip_lock:
|
||||
if worker.ip_address == ip_address:
|
||||
if worker.error_status == 'ip_lock':
|
||||
worker.error_status = ''
|
||||
worker.save()
|
||||
else:
|
||||
worker.error_status = 'ip_lock'
|
||||
worker.save()
|
||||
success = False
|
||||
else:
|
||||
if worker.ip_address != ip_address:
|
||||
worker.ip_address = ip_address
|
||||
worker.save()
|
||||
|
||||
if worker.enabled:
|
||||
if worker.error_status == 'worker_disabled':
|
||||
worker.error_status = ''
|
||||
worker.save()
|
||||
else:
|
||||
worker.error_status = 'worker_disabled'
|
||||
worker.save()
|
||||
success = False
|
||||
|
||||
cluster_settings, created = ClusterSettings.objects.get_or_create(name='cluster_settings')
|
||||
if not settings.WIREGUARD_STATUS_CACHE_ENABLED and cluster_settings.enabled:
|
||||
cluster_settings.enabled = False
|
||||
cluster_settings.save()
|
||||
|
||||
if cluster_settings.enabled:
|
||||
if worker.error_status == 'cluster_disabled':
|
||||
worker.error_status = ''
|
||||
worker.save()
|
||||
else:
|
||||
worker.error_status = 'cluster_disabled'
|
||||
worker.save()
|
||||
success = False
|
||||
|
||||
return worker, success
|
||||
if error_status:
|
||||
return None, _error_response(error_status, 400)
|
||||
return worker, None
|
||||
|
||||
|
||||
def api_get_worker_dnsmasq_config(request):
|
||||
dnsmasq_file = "/etc/dnsmasq/dnsmasq_config.tar.gz"
|
||||
worker, success = get_worker(request)
|
||||
if worker:
|
||||
if worker.error_status or not success:
|
||||
data = {'status': 'error', 'message': worker.error_status}
|
||||
return JsonResponse(data, status=400)
|
||||
else:
|
||||
data = {'status': 'error', 'message': 'Worker not found'}
|
||||
return JsonResponse(data, status=403)
|
||||
def api_legacy_cluster_status(request):
|
||||
# Workers before version 12 poll this endpoint. A 403 makes them shut down WireGuard and stop polling.
|
||||
return JsonResponse({
|
||||
'status': 'error',
|
||||
'message': 'update_required',
|
||||
'minimum_worker_version': settings.CLUSTER_WORKER_MINIMUM_VERSION,
|
||||
}, status=403)
|
||||
|
||||
if not os.path.exists(dnsmasq_file):
|
||||
data = {'status': 'error', 'message': 'dnsmasq configuration not found'}
|
||||
return JsonResponse(data, status=404)
|
||||
|
||||
response = FileResponse(open(dnsmasq_file, "rb"), content_type="application/gzip")
|
||||
response["Content-Disposition"] = 'attachment; filename="dnsmasq_config.tar.gz"'
|
||||
response["Content-Length"] = str(os.path.getsize(dnsmasq_file))
|
||||
return response
|
||||
@require_GET
|
||||
def api_manifest(request):
|
||||
worker, error_response = get_worker(request)
|
||||
if error_response:
|
||||
return error_response
|
||||
return JsonResponse({
|
||||
'status': 'success',
|
||||
'worker_minimum_version': settings.CLUSTER_WORKER_MINIMUM_VERSION,
|
||||
'sections': build_manifest(worker),
|
||||
})
|
||||
|
||||
|
||||
@gzip_page
|
||||
@require_GET
|
||||
def api_section(request, section_name):
|
||||
worker, error_response = get_worker(request)
|
||||
if error_response:
|
||||
return error_response
|
||||
if section_name not in SECTION_NAMES:
|
||||
return _error_response('Section not found', 404)
|
||||
return JsonResponse({'status': 'success', 'section': section_name, **build_section(section_name, worker).as_dict()})
|
||||
|
||||
|
||||
def _clean_section_report(sections):
|
||||
cleaned_sections = {}
|
||||
for section_name, report in sections.items():
|
||||
if section_name not in SECTION_NAMES or not isinstance(report, dict):
|
||||
continue
|
||||
status = report.get('status')
|
||||
if status not in ('ok', 'error'):
|
||||
continue
|
||||
cleaned_sections[section_name] = {
|
||||
'hash': str(report.get('hash', ''))[:64],
|
||||
'status': status,
|
||||
'message': str(report.get('message', ''))[:1000],
|
||||
'updated': timezone.now().isoformat(),
|
||||
}
|
||||
return cleaned_sections
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
def api_submit_worker_wireguard_stats(request):
|
||||
worker, success = get_worker(request)
|
||||
if worker:
|
||||
if worker.error_status or not success:
|
||||
data = {'status': 'error', 'message': worker.error_status}
|
||||
return JsonResponse(data, status=400)
|
||||
else:
|
||||
data = {'status': 'error', 'message': 'Worker not found'}
|
||||
return JsonResponse(data, status=403)
|
||||
|
||||
@require_POST
|
||||
def api_report(request):
|
||||
worker, error_response = get_worker(request)
|
||||
if error_response:
|
||||
return error_response
|
||||
try:
|
||||
if request.method == 'POST':
|
||||
payload = json.loads(request.body)
|
||||
worker_status = worker.workerstatus
|
||||
worker_status.wireguard_status = payload
|
||||
worker_status.wireguard_status_updated = timezone.now()
|
||||
worker_status.save()
|
||||
data = {'status': 'success', 'message': 'Stats received'}
|
||||
return JsonResponse(data, status=200)
|
||||
except Exception as e:
|
||||
pass
|
||||
payload = json.loads(request.body)
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
return _error_response('Invalid JSON', 400)
|
||||
if not isinstance(payload, dict):
|
||||
return _error_response('Invalid JSON', 400)
|
||||
|
||||
data = {'status': 'error', 'message': 'Stats not received'}
|
||||
return JsonResponse(data, status=400)
|
||||
worker_status = worker.workerstatus
|
||||
update_fields = []
|
||||
|
||||
if isinstance(payload.get('sections'), dict):
|
||||
section_status = dict(worker_status.section_status or {})
|
||||
section_status.update(_clean_section_report(payload['sections']))
|
||||
worker_status.section_status = section_status
|
||||
update_fields.append('section_status')
|
||||
|
||||
def api_get_worker_config_files(request):
|
||||
worker, success = get_worker(request)
|
||||
if worker:
|
||||
if worker.error_status or not success:
|
||||
data = {'status': 'error', 'message': worker.error_status}
|
||||
return JsonResponse(data, status=400)
|
||||
else:
|
||||
data = {'status': 'error', 'message': 'Worker not found'}
|
||||
return JsonResponse(data, status=403)
|
||||
if isinstance(payload.get('services'), dict):
|
||||
worker_status.services_status = payload['services']
|
||||
update_fields.append('services_status')
|
||||
|
||||
config_files = (
|
||||
glob.glob('/etc/wireguard/wg*.conf') +
|
||||
glob.glob('/etc/wireguard/wg-firewall.sh')
|
||||
)
|
||||
if isinstance(payload.get('certificates'), dict):
|
||||
worker_status.certificate_status = payload['certificates']
|
||||
worker_status.certificate_status_updated = timezone.now()
|
||||
update_fields += ['certificate_status', 'certificate_status_updated']
|
||||
|
||||
files = {}
|
||||
if isinstance(payload.get('wireguard_status'), dict):
|
||||
worker_status.wireguard_status = payload['wireguard_status']
|
||||
worker_status.wireguard_status_updated = timezone.now()
|
||||
update_fields += ['wireguard_status', 'wireguard_status_updated']
|
||||
|
||||
for path in config_files:
|
||||
filename = os.path.basename(path)
|
||||
with open(path, 'r') as f:
|
||||
files[filename] = f.read()
|
||||
|
||||
data = {'status': 'success', 'files': files, 'cluster_settings': get_cluster_settings()}
|
||||
return JsonResponse(data, status=200)
|
||||
|
||||
|
||||
def api_worker_ping(request):
|
||||
worker, success = get_worker(request)
|
||||
if worker:
|
||||
if worker.error_status or not success:
|
||||
data = {'status': 'error', 'message': worker.error_status}
|
||||
return JsonResponse(data, status=400)
|
||||
else:
|
||||
data = {'status': 'error', 'message': 'Worker not found'}
|
||||
return JsonResponse(data, status=403)
|
||||
|
||||
data = {
|
||||
'status': 'success',
|
||||
'worker_error_status': worker.error_status,
|
||||
}
|
||||
|
||||
return JsonResponse(data, status=200)
|
||||
|
||||
|
||||
def api_cluster_status(request):
|
||||
worker, success = get_worker(request)
|
||||
if worker:
|
||||
if worker.error_status or not success:
|
||||
data = {'status': 'error', 'message': worker.error_status}
|
||||
return JsonResponse(data, status=400)
|
||||
else:
|
||||
data = {'status': 'error', 'message': 'Worker not found'}
|
||||
return JsonResponse(data, status=403)
|
||||
data = {'status': 'success', 'worker_error_status': worker.error_status, 'cluster_settings': get_cluster_settings()}
|
||||
return JsonResponse(data, status=200)
|
||||
if update_fields:
|
||||
worker_status.save(update_fields=update_fields + ['updated'])
|
||||
return JsonResponse({'status': 'success'})
|
||||
|
||||
+5
-8
@@ -31,8 +31,10 @@ class WorkerForm(forms.ModelForm):
|
||||
|
||||
if self.instance.pk:
|
||||
delete_html = f"<a href='javascript:void(0)' class='btn btn-outline-danger' data-command='delete' onclick='openCommandDialog(this)'>{delete_label}</a>"
|
||||
back_url = f"/cluster/worker/view/?uuid={self.instance.uuid}"
|
||||
else:
|
||||
delete_html = ''
|
||||
back_url = '/cluster/'
|
||||
|
||||
self.helper.layout = Layout(
|
||||
Row(
|
||||
@@ -65,7 +67,7 @@ class WorkerForm(forms.ModelForm):
|
||||
Row(
|
||||
Column(
|
||||
Submit('submit', _('Save'), css_class='btn btn-success'),
|
||||
HTML(f' <a class="btn btn-secondary" href="/cluster/">{back_label}</a> '),
|
||||
HTML(f' <a class="btn btn-secondary" href="{back_url}">{back_label}</a> '),
|
||||
HTML(delete_html),
|
||||
css_class='col-md-12'),
|
||||
css_class='form-row'
|
||||
@@ -90,14 +92,13 @@ class WorkerForm(forms.ModelForm):
|
||||
class ClusterSettingsForm(forms.ModelForm):
|
||||
class Meta:
|
||||
model = ClusterSettings
|
||||
fields = ['enabled', 'primary_enable_wireguard', 'cluster_mode', 'restart_mode', 'worker_display']
|
||||
fields = ['enabled', 'primary_enable_wireguard', 'cluster_mode', 'worker_display']
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.fields['enabled'].label = _("Cluster Enabled")
|
||||
self.fields['primary_enable_wireguard'].label = _("Primary Enable WireGuard")
|
||||
self.fields['cluster_mode'].label = _("Cluster Mode")
|
||||
self.fields['restart_mode'].label = _("Restart Mode")
|
||||
self.fields['worker_display'].label = _("Worker Display")
|
||||
|
||||
back_label = _("Back")
|
||||
@@ -112,11 +113,7 @@ class ClusterSettingsForm(forms.ModelForm):
|
||||
),
|
||||
Row(
|
||||
Column('cluster_mode', css_class='form-group col-md-6 mb-0'),
|
||||
Column('restart_mode', css_class='form-group col-md-6 mb-0'),
|
||||
css_class='form-row'
|
||||
),
|
||||
Row(
|
||||
Column('worker_display', css_class='form-group col-md-12 mb-0'),
|
||||
Column('worker_display', css_class='form-group col-md-6 mb-0'),
|
||||
css_class='form-row'
|
||||
),
|
||||
Row(
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
# Generated by Django 5.2.19 on 2026-11-19 16:11
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('cluster', '0017_alter_workerstatus_wireguard_status'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RemoveField(
|
||||
model_name='clustersettings',
|
||||
name='config_version',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='clustersettings',
|
||||
name='dns_version',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='clustersettings',
|
||||
name='restart_mode',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='worker',
|
||||
name='force_reload',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='worker',
|
||||
name='force_restart',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='workerstatus',
|
||||
name='active_peers',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='workerstatus',
|
||||
name='config_version',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='workerstatus',
|
||||
name='dns_version',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='workerstatus',
|
||||
name='last_reload',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='workerstatus',
|
||||
name='last_restart',
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='workerstatus',
|
||||
name='certificate_status',
|
||||
field=models.JSONField(blank=True, default=dict),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='workerstatus',
|
||||
name='certificate_status_updated',
|
||||
field=models.DateTimeField(blank=True, null=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='workerstatus',
|
||||
name='section_status',
|
||||
field=models.JSONField(blank=True, default=dict),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='workerstatus',
|
||||
name='services_status',
|
||||
field=models.JSONField(blank=True, default=dict),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='worker',
|
||||
name='error_status',
|
||||
field=models.CharField(choices=[('', ''), ('ip_lock', 'IP lock is enabled, but the worker is attempting to access from a different IP address.'), ('worker_disabled', 'Worker is not enabled'), ('cluster_disabled', 'Cluster is not enabled'), ('update_required', 'Worker update is required.')], default='', max_length=32),
|
||||
),
|
||||
]
|
||||
+10
-12
@@ -9,15 +9,12 @@ class ClusterSettings(models.Model):
|
||||
enabled = models.BooleanField(default=False)
|
||||
primary_enable_wireguard = models.BooleanField(default=True)
|
||||
cluster_mode = models.CharField(default='mirror', max_length=16, choices=(('mirror', 'Mirror'), ))
|
||||
restart_mode = models.CharField(default='auto', max_length=16, choices=(('auto', 'Automatic restart'),))
|
||||
worker_display = models.CharField(
|
||||
default='server_address', max_length=16, choices=(
|
||||
('name', 'Name'), ('server_address', 'Server Address'),
|
||||
('location', 'Location'), ('address_location', 'Address + Location')
|
||||
)
|
||||
)
|
||||
config_version = models.PositiveIntegerField(default=0)
|
||||
dns_version = models.PositiveIntegerField(default=0)
|
||||
|
||||
updated = models.DateTimeField(auto_now=True)
|
||||
created = models.DateTimeField(auto_now_add=True)
|
||||
@@ -26,6 +23,10 @@ class ClusterSettings(models.Model):
|
||||
def __str__(self):
|
||||
return self.name
|
||||
|
||||
@classmethod
|
||||
def cluster_enabled(cls):
|
||||
return cls.objects.filter(name='cluster_settings', enabled=True).exists()
|
||||
|
||||
|
||||
class Worker(models.Model):
|
||||
name = models.CharField(max_length=100, unique=True)
|
||||
@@ -34,9 +35,6 @@ class Worker(models.Model):
|
||||
ip_lock = models.BooleanField(default=False)
|
||||
ip_address = models.GenericIPAddressField(blank=True, null=True)
|
||||
|
||||
force_reload = models.BooleanField(default=False)
|
||||
force_restart = models.BooleanField(default=False)
|
||||
|
||||
country = models.CharField(max_length=100, blank=True, null=True)
|
||||
city = models.CharField(max_length=100, blank=True, null=True)
|
||||
hostname = models.CharField(max_length=100, blank=True, null=True)
|
||||
@@ -46,7 +44,6 @@ class Worker(models.Model):
|
||||
('ip_lock', _('IP lock is enabled, but the worker is attempting to access from a different IP address.')),
|
||||
('worker_disabled', _('Worker is not enabled')),
|
||||
('cluster_disabled', _('Cluster is not enabled')),
|
||||
('missing_version', _('Please report worker_config_version, worker_dns_version and worker_version in the API request.')),
|
||||
('update_required', _('Worker update is required.'))
|
||||
))
|
||||
|
||||
@@ -105,14 +102,15 @@ class Worker(models.Model):
|
||||
class WorkerStatus(models.Model):
|
||||
worker = models.OneToOneField(Worker, on_delete=models.CASCADE)
|
||||
last_seen = models.DateTimeField(auto_now=True)
|
||||
last_reload = models.DateTimeField(blank=True, null=True)
|
||||
last_restart = models.DateTimeField(blank=True, null=True)
|
||||
dns_version = models.PositiveIntegerField(default=0)
|
||||
config_version = models.PositiveIntegerField(default=0)
|
||||
worker_version = models.PositiveIntegerField(default=0)
|
||||
active_peers = models.PositiveIntegerField(default=0)
|
||||
wireguard_status = models.JSONField(blank=True, null=True)
|
||||
wireguard_status_updated = models.DateTimeField(blank=True, null=True)
|
||||
# {section: {"hash": ..., "status": "ok"|"error", "message": ..., "updated": ...}}
|
||||
section_status = models.JSONField(default=dict, blank=True)
|
||||
services_status = models.JSONField(default=dict, blank=True)
|
||||
# {hostname: {"status": ..., "issuer": ..., "not_after": ..., "generation": ...}}
|
||||
certificate_status = models.JSONField(default=dict, blank=True)
|
||||
certificate_status_updated = models.DateTimeField(blank=True, null=True)
|
||||
|
||||
updated = models.DateTimeField(auto_now=True)
|
||||
created = models.DateTimeField(auto_now_add=True)
|
||||
|
||||
+209
@@ -0,0 +1,209 @@
|
||||
import glob
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from datetime import timedelta
|
||||
|
||||
from django.conf import settings
|
||||
from django.utils import timezone
|
||||
from django.utils.dateparse import parse_datetime
|
||||
|
||||
from app_gateway.caddy_config_export import (
|
||||
add_certificate_files, certificate_file_name, load_worker_export, read_exported_certificate
|
||||
)
|
||||
from cluster.models import WorkerStatus
|
||||
from dns.functions import generate_dnsmasq_config
|
||||
from dns.models import DNSFilterList
|
||||
from firewall.tools import generate_firewall_script
|
||||
|
||||
WIREGUARD_CONFIG_DIR = '/etc/wireguard'
|
||||
DNSMASQ_CONFIG_DIR = '/etc/dnsmasq'
|
||||
DNSMASQ_MAIN_CONFIG = 'wireguard_webadmin_dns.conf'
|
||||
|
||||
# Sections are applied by the node in this order. The firewall comes first because the WireGuard
|
||||
# configuration runs wg-firewall.sh as PostUp of one of the instances, and certificates come before the
|
||||
# gateway so the generated Caddyfile only references files that already exist.
|
||||
SECTION_NAMES = ('firewall', 'wireguard', 'dns', 'settings', 'certificates', 'gateway')
|
||||
|
||||
# Large files (e.g. DNS filter lists) are only read and hashed again when they change on disk
|
||||
_file_cache = {}
|
||||
|
||||
|
||||
def _sha256(text):
|
||||
return hashlib.sha256(text.encode()).hexdigest()
|
||||
|
||||
|
||||
def _read_file(path):
|
||||
file_stat = os.stat(path)
|
||||
cache_key = (file_stat.st_mtime_ns, file_stat.st_size)
|
||||
cached = _file_cache.get(path)
|
||||
if cached and cached['key'] == cache_key:
|
||||
return cached['content'], cached['sha256']
|
||||
with open(path, 'r', encoding='utf-8', errors='replace') as config_file:
|
||||
content = config_file.read()
|
||||
content_sha256 = _sha256(content)
|
||||
_file_cache[path] = {'key': cache_key, 'content': content, 'sha256': content_sha256}
|
||||
return content, content_sha256
|
||||
|
||||
|
||||
class Section:
|
||||
def __init__(self):
|
||||
self.files = {}
|
||||
self.file_hashes = {}
|
||||
self.values = {}
|
||||
|
||||
def add_file(self, filename, content, content_sha256=None):
|
||||
self.files[filename] = content
|
||||
self.file_hashes[filename] = content_sha256 or _sha256(content)
|
||||
|
||||
@property
|
||||
def hash(self):
|
||||
fingerprint = json.dumps({'files': self.file_hashes, 'values': self.values}, sort_keys=True)
|
||||
return _sha256(fingerprint)
|
||||
|
||||
def as_dict(self):
|
||||
return {'hash': self.hash, 'files': self.files, 'values': self.values}
|
||||
|
||||
|
||||
def build_wireguard_section(worker):
|
||||
section = Section()
|
||||
for config_path in sorted(glob.glob(os.path.join(WIREGUARD_CONFIG_DIR, 'wg[0-9]*.conf'))):
|
||||
content, content_sha256 = _read_file(config_path)
|
||||
section.add_file(os.path.basename(config_path), content, content_sha256)
|
||||
return section
|
||||
|
||||
|
||||
def build_firewall_section(worker):
|
||||
section = Section()
|
||||
section.add_file('wg-firewall.sh', generate_firewall_script(cluster_node=True))
|
||||
return section
|
||||
|
||||
|
||||
def build_dns_section(worker):
|
||||
section = Section()
|
||||
for dns_list in DNSFilterList.objects.filter(enabled=True).order_by('uuid'):
|
||||
list_path = os.path.join(DNSMASQ_CONFIG_DIR, f'{dns_list.uuid}.conf')
|
||||
if os.path.isfile(list_path):
|
||||
content, content_sha256 = _read_file(list_path)
|
||||
section.add_file(os.path.basename(list_path), content, content_sha256)
|
||||
section.add_file(DNSMASQ_MAIN_CONFIG, generate_dnsmasq_config(cluster_node=True))
|
||||
return section
|
||||
|
||||
|
||||
def build_settings_section(worker):
|
||||
section = Section()
|
||||
section.values = {
|
||||
'stats_sync_interval': settings.WIREGUARD_STATUS_CACHE_REFRESH_INTERVAL,
|
||||
}
|
||||
return section
|
||||
|
||||
|
||||
EMPTY_GATEWAY_EXPORT = {
|
||||
'applications.json': {'entries': []},
|
||||
'auth_policies.json': {'auth_methods': {}, 'groups': {}, 'users': {}, 'policies': {}},
|
||||
'routes.json': {'entries': {}},
|
||||
'tls.json': {'hosts': {}, 'dns_credentials': {}},
|
||||
}
|
||||
|
||||
|
||||
def get_gateway_export(worker):
|
||||
"""Configuration exported for the worker, with the certificate files currently available on the primary."""
|
||||
gateway_export = load_worker_export(worker) or json.loads(json.dumps(EMPTY_GATEWAY_EXPORT))
|
||||
add_certificate_files(gateway_export['tls.json'])
|
||||
return gateway_export
|
||||
|
||||
|
||||
def build_certificates_section(worker):
|
||||
"""Certificate and private key of every ACME host served by the worker, named after the certificate hash."""
|
||||
section = Section()
|
||||
for hostname, host_tls in sorted(get_gateway_export(worker)['tls.json']['hosts'].items()):
|
||||
if not host_tls.get('certificate_file'):
|
||||
continue
|
||||
certificate, private_key = read_exported_certificate(hostname)
|
||||
if not certificate or not private_key:
|
||||
continue
|
||||
file_name = certificate_file_name(certificate)
|
||||
section.add_file(f'{file_name}.crt', certificate)
|
||||
section.add_file(f'{file_name}.key', private_key)
|
||||
return section
|
||||
|
||||
|
||||
def build_gateway_section(worker):
|
||||
section = Section()
|
||||
for file_name, data in get_gateway_export(worker).items():
|
||||
section.add_file(file_name, json.dumps(data, indent=2) + '\n')
|
||||
return section
|
||||
|
||||
|
||||
SECTION_BUILDERS = {
|
||||
'firewall': build_firewall_section,
|
||||
'wireguard': build_wireguard_section,
|
||||
'dns': build_dns_section,
|
||||
'settings': build_settings_section,
|
||||
'certificates': build_certificates_section,
|
||||
'gateway': build_gateway_section,
|
||||
}
|
||||
|
||||
|
||||
def build_section(section_name, worker):
|
||||
return SECTION_BUILDERS[section_name](worker)
|
||||
|
||||
|
||||
def build_manifest(worker):
|
||||
return {section_name: build_section(section_name, worker).hash for section_name in SECTION_NAMES}
|
||||
|
||||
|
||||
CERTIFICATE_EXPIRATION_WARNING_DAYS = 14
|
||||
|
||||
|
||||
def get_worker_warnings(worker):
|
||||
"""Stopped services and certificates that are not trusted or expire soon, as reported by the worker."""
|
||||
try:
|
||||
worker_status = worker.workerstatus
|
||||
except WorkerStatus.DoesNotExist:
|
||||
return {'stopped_services': [], 'certificate_issues': []}
|
||||
|
||||
stopped_services = sorted(
|
||||
service for service, state in (worker_status.services_status or {}).items() if state != 'running'
|
||||
)
|
||||
expiration_limit = timezone.now() + timedelta(days=CERTIFICATE_EXPIRATION_WARNING_DAYS)
|
||||
certificate_issues = []
|
||||
for hostname, certificate in sorted((worker_status.certificate_status or {}).items()):
|
||||
not_after = parse_datetime(certificate.get('not_after') or '')
|
||||
if certificate.get('status') not in ('trusted', 'untrusted'):
|
||||
certificate_issues.append(hostname)
|
||||
elif certificate.get('status') == 'trusted' and not_after and not_after < expiration_limit:
|
||||
certificate_issues.append(hostname)
|
||||
return {'stopped_services': stopped_services, 'certificate_issues': certificate_issues}
|
||||
|
||||
|
||||
def get_worker_sync_state(worker, manifest=None):
|
||||
"""
|
||||
Compare the hashes reported by the worker with the current ones.
|
||||
Returns {'state': 'synced'|'pending'|'error', 'sections': {name: {'state': ..., 'message': ...}}}.
|
||||
"""
|
||||
manifest = manifest or build_manifest(worker)
|
||||
try:
|
||||
section_status = worker.workerstatus.section_status or {}
|
||||
except WorkerStatus.DoesNotExist:
|
||||
section_status = {}
|
||||
|
||||
sections = {}
|
||||
for section_name, section_hash in manifest.items():
|
||||
reported = section_status.get(section_name) or {}
|
||||
if reported.get('hash') != section_hash:
|
||||
# A previous error is kept as information while the node retries with the new content
|
||||
sections[section_name] = {'state': 'pending', 'message': reported.get('message', '')}
|
||||
elif reported.get('status') == 'error':
|
||||
sections[section_name] = {'state': 'error', 'message': reported.get('message', '')}
|
||||
else:
|
||||
sections[section_name] = {'state': 'synced', 'message': ''}
|
||||
|
||||
states = {section['state'] for section in sections.values()}
|
||||
if 'error' in states:
|
||||
state = 'error'
|
||||
elif 'pending' in states:
|
||||
state = 'pending'
|
||||
else:
|
||||
state = 'synced'
|
||||
return {'state': state, 'sections': sections}
|
||||
+273
-1
@@ -1 +1,273 @@
|
||||
# Create your tests here.
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from unittest.mock import patch
|
||||
|
||||
from django.test import TestCase, override_settings
|
||||
from django.utils import timezone
|
||||
|
||||
from api.models import WireguardStatusCache
|
||||
from api.views import func_concatenate_cluster_wireguard_status_cache
|
||||
from app_gateway import caddy_config_export
|
||||
from app_gateway.caddy_config_export import certificate_file_name, export_caddy_config
|
||||
from app_gateway.models import Application, ApplicationHost, DnsChallengeCredential
|
||||
from cluster import sync
|
||||
from cluster.models import ClusterSettings, Worker, WorkerStatus
|
||||
from dns.models import DNSSettings
|
||||
from firewall.tools import generate_firewall_script
|
||||
|
||||
FAKE_CERTIFICATE = '-----BEGIN CERTIFICATE-----\nfake\n-----END CERTIFICATE-----\n'
|
||||
FAKE_PRIVATE_KEY = '-----BEGIN PRIVATE KEY-----\nfake\n-----END PRIVATE KEY-----\n'
|
||||
|
||||
|
||||
class ClusterTestMixin:
|
||||
"""Isolated directories for the files read and written by the cluster sync and the Caddy export."""
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.temp_dir = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp_dir.cleanup)
|
||||
self.wireguard_dir = os.path.join(self.temp_dir.name, 'wireguard')
|
||||
self.dnsmasq_dir = os.path.join(self.temp_dir.name, 'dnsmasq')
|
||||
self.export_dir = os.path.join(self.temp_dir.name, 'caddy_json_export')
|
||||
self.certificate_export_dir = os.path.join(self.temp_dir.name, 'caddy_cert_export')
|
||||
for directory in (self.wireguard_dir, self.dnsmasq_dir, self.export_dir, self.certificate_export_dir):
|
||||
os.makedirs(directory)
|
||||
with open(os.path.join(self.wireguard_dir, 'wg0.conf'), 'w', encoding='utf-8') as config_file:
|
||||
config_file.write('[Interface]\nListenPort = 51820\n')
|
||||
|
||||
for target, value in (
|
||||
('cluster.sync.WIREGUARD_CONFIG_DIR', self.wireguard_dir),
|
||||
('cluster.sync.DNSMASQ_CONFIG_DIR', self.dnsmasq_dir),
|
||||
('app_gateway.caddy_config_export.CADDY_EXPORT_DIR', self.export_dir),
|
||||
('app_gateway.caddy_config_export.CERTIFICATE_EXPORT_DIR', self.certificate_export_dir),
|
||||
):
|
||||
patcher = patch(target, value)
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
DNSSettings.objects.get_or_create(name='dns_settings')
|
||||
ClusterSettings.objects.update_or_create(name='cluster_settings', defaults={'enabled': True})
|
||||
self.worker = Worker.objects.create(name='node-a', hostname='node-a.example.test')
|
||||
self.headers = {'HTTP_AUTHORIZATION': f'Bearer {self.worker.token}', 'HTTP_X_WORKER_VERSION': '12'}
|
||||
|
||||
def write_exported_certificate(self, hostname):
|
||||
host_dir = os.path.join(self.certificate_export_dir, hostname)
|
||||
os.makedirs(host_dir, exist_ok=True)
|
||||
with open(os.path.join(host_dir, 'certificate.crt'), 'w', encoding='utf-8') as certificate_file:
|
||||
certificate_file.write(FAKE_CERTIFICATE)
|
||||
with open(os.path.join(host_dir, 'certificate.key'), 'w', encoding='utf-8') as key_file:
|
||||
key_file.write(FAKE_PRIVATE_KEY)
|
||||
|
||||
|
||||
@override_settings(CLUSTER_WORKER_MINIMUM_VERSION=12, WIREGUARD_STATUS_CACHE_ENABLED=True)
|
||||
class ClusterApiTests(ClusterTestMixin, TestCase):
|
||||
def test_legacy_status_endpoint_stops_old_workers_without_token(self):
|
||||
response = self.client.get('/api/cluster/status/')
|
||||
self.assertEqual(response.status_code, 403)
|
||||
self.assertEqual(response.json()['message'], 'update_required')
|
||||
self.assertEqual(response.json()['minimum_worker_version'], 12)
|
||||
|
||||
def test_manifest_requires_token(self):
|
||||
self.assertEqual(self.client.get('/api/cluster/v2/manifest/').status_code, 403)
|
||||
response = self.client.get('/api/cluster/v2/manifest/', HTTP_AUTHORIZATION='Bearer not-a-uuid')
|
||||
self.assertEqual(response.status_code, 403)
|
||||
|
||||
def test_worker_is_refused_with_reason(self):
|
||||
scenarios = (
|
||||
({'HTTP_X_WORKER_VERSION': '11'}, {}, 'update_required'),
|
||||
({}, {'enabled': False}, 'worker_disabled'),
|
||||
({}, {'ip_lock': True, 'ip_address': '203.0.113.10'}, 'ip_lock'),
|
||||
)
|
||||
for header_override, worker_fields, expected_error in scenarios:
|
||||
with self.subTest(expected_error=expected_error):
|
||||
Worker.objects.filter(pk=self.worker.pk).update(**{'enabled': True, 'ip_lock': False, **worker_fields})
|
||||
response = self.client.get('/api/cluster/v2/manifest/', **{**self.headers, **header_override})
|
||||
self.assertEqual(response.status_code, 400)
|
||||
self.assertEqual(response.json()['message'], expected_error)
|
||||
self.worker.refresh_from_db()
|
||||
self.assertEqual(self.worker.error_status, expected_error)
|
||||
|
||||
def test_disabled_cluster_refuses_workers(self):
|
||||
ClusterSettings.objects.filter(name='cluster_settings').update(enabled=False)
|
||||
response = self.client.get('/api/cluster/v2/manifest/', **self.headers)
|
||||
self.assertEqual(response.status_code, 400)
|
||||
self.assertEqual(response.json()['message'], 'cluster_disabled')
|
||||
|
||||
def test_manifest_hashes_match_sections(self):
|
||||
response = self.client.get('/api/cluster/v2/manifest/', **self.headers)
|
||||
self.assertEqual(response.status_code, 200)
|
||||
manifest = response.json()['sections']
|
||||
self.assertEqual(list(manifest), list(sync.SECTION_NAMES))
|
||||
for section_name, section_hash in manifest.items():
|
||||
with self.subTest(section=section_name):
|
||||
section_response = self.client.get(f'/api/cluster/v2/section/{section_name}/', **self.headers)
|
||||
self.assertEqual(section_response.status_code, 200)
|
||||
self.assertEqual(section_response.json()['hash'], section_hash)
|
||||
self.assertEqual(self.client.get('/api/cluster/v2/section/unknown/', **self.headers).status_code, 404)
|
||||
|
||||
def test_report_stores_worker_status(self):
|
||||
report = {
|
||||
'sections': {
|
||||
'dns': {'hash': 'abc', 'status': 'error', 'message': 'dnsmasq failed'},
|
||||
'unknown': {'hash': 'abc', 'status': 'ok'},
|
||||
'settings': {'hash': 'def', 'status': 'invalid'},
|
||||
},
|
||||
'services': {'dnsmasq': 'running'},
|
||||
'certificates': {'app.example.test': {'status': 'trusted'}},
|
||||
'wireguard_status': {'wg0': {}},
|
||||
}
|
||||
response = self.client.post(
|
||||
'/api/cluster/v2/report/', data=json.dumps(report), content_type='application/json', **self.headers
|
||||
)
|
||||
self.assertEqual(response.status_code, 200)
|
||||
worker_status = WorkerStatus.objects.get(worker=self.worker)
|
||||
self.assertEqual(list(worker_status.section_status), ['dns'])
|
||||
self.assertEqual(worker_status.section_status['dns']['message'], 'dnsmasq failed')
|
||||
self.assertEqual(worker_status.services_status, {'dnsmasq': 'running'})
|
||||
self.assertEqual(worker_status.certificate_status, report['certificates'])
|
||||
self.assertEqual(worker_status.wireguard_status, {'wg0': {}})
|
||||
self.assertIsNotNone(worker_status.certificate_status_updated)
|
||||
self.assertIsNotNone(worker_status.wireguard_status_updated)
|
||||
|
||||
|
||||
class ClusterSyncTests(ClusterTestMixin, TestCase):
|
||||
def report_sections(self, manifest, overrides=None):
|
||||
section_status = {name: {'hash': section_hash, 'status': 'ok'} for name, section_hash in manifest.items()}
|
||||
section_status.update(overrides or {})
|
||||
WorkerStatus.objects.update_or_create(worker=self.worker, defaults={'section_status': section_status})
|
||||
self.worker.refresh_from_db()
|
||||
|
||||
def test_sync_state(self):
|
||||
manifest = sync.build_manifest(self.worker)
|
||||
self.assertEqual(sync.get_worker_sync_state(self.worker, manifest)['state'], 'pending')
|
||||
|
||||
self.report_sections(manifest)
|
||||
self.assertEqual(sync.get_worker_sync_state(self.worker, manifest)['state'], 'synced')
|
||||
|
||||
self.report_sections(manifest, {'dns': {'hash': manifest['dns'], 'status': 'error', 'message': 'failed'}})
|
||||
sync_state = sync.get_worker_sync_state(self.worker, manifest)
|
||||
self.assertEqual(sync_state['state'], 'error')
|
||||
self.assertEqual(sync_state['sections']['dns']['message'], 'failed')
|
||||
|
||||
# An error reported for older content means the node still has to try the new one
|
||||
self.report_sections(manifest, {'dns': {'hash': 'old', 'status': 'error', 'message': 'failed'}})
|
||||
self.assertEqual(sync.get_worker_sync_state(self.worker, manifest)['state'], 'pending')
|
||||
|
||||
def test_wireguard_section_follows_config_files(self):
|
||||
first_hash = sync.build_section('wireguard', self.worker).hash
|
||||
self.assertEqual(sync.build_section('wireguard', self.worker).hash, first_hash)
|
||||
with open(os.path.join(self.wireguard_dir, 'wg1.conf'), 'w', encoding='utf-8') as config_file:
|
||||
config_file.write('[Interface]\nListenPort = 51821\n')
|
||||
section = sync.build_section('wireguard', self.worker)
|
||||
self.assertNotEqual(section.hash, first_hash)
|
||||
self.assertEqual(sorted(section.files), ['wg0.conf', 'wg1.conf'])
|
||||
|
||||
def test_dns_section_uses_node_configuration(self):
|
||||
section = sync.build_section('dns', self.worker)
|
||||
main_config = section.files[sync.DNSMASQ_MAIN_CONFIG]
|
||||
self.assertIn('bind-dynamic', main_config)
|
||||
self.assertIn('interface=wg*', main_config)
|
||||
self.assertNotIn('listen-address=0.0.0.0', main_config)
|
||||
|
||||
def test_firewall_section_is_stable_and_node_specific(self):
|
||||
self.assertEqual(sync.build_section('firewall', self.worker).hash, sync.build_section('firewall', self.worker).hash)
|
||||
node_script = generate_firewall_script(cluster_node=True)
|
||||
primary_script = generate_firewall_script()
|
||||
self.assertNotIn('DNS_IP', node_script)
|
||||
self.assertNotIn('--dport 8000', node_script)
|
||||
self.assertNotIn('generated by', node_script)
|
||||
self.assertIn('DNS_IP', primary_script)
|
||||
|
||||
|
||||
class ClusterGatewaySyncTests(ClusterTestMixin, TestCase):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.credential = DnsChallengeCredential.objects.create(
|
||||
display_name='Cloudflare', provider='cloudflare', credentials={'api_token': 'x' * 40}
|
||||
)
|
||||
self.node_app = Application.objects.create(
|
||||
display_name='Node App', upstream='http://10.0.0.20:8080', serve_on_primary=False
|
||||
)
|
||||
self.node_app.workers.add(self.worker)
|
||||
ApplicationHost.objects.create(
|
||||
application=self.node_app, hostname='node.example.test', certificate_type='dns_challenge',
|
||||
dns_credential=self.credential,
|
||||
)
|
||||
|
||||
def gateway_files(self):
|
||||
return {
|
||||
filename: json.loads(content)
|
||||
for filename, content in sync.build_section('gateway', self.worker).files.items()
|
||||
}
|
||||
|
||||
def test_primary_manages_certificates_of_worker_only_applications(self):
|
||||
export_caddy_config(self.export_dir)
|
||||
with open(os.path.join(self.export_dir, 'applications.json'), encoding='utf-8') as applications_file:
|
||||
primary_applications = json.load(applications_file)
|
||||
with open(os.path.join(self.export_dir, 'tls.json'), encoding='utf-8') as tls_file:
|
||||
primary_tls = json.load(tls_file)
|
||||
self.assertNotIn('node-app', [entry['id'] for entry in primary_applications['entries']])
|
||||
host_tls = primary_tls['hosts']['node.example.test']
|
||||
self.assertTrue(host_tls['certificate_only'])
|
||||
self.assertTrue(host_tls['export_certificate'])
|
||||
self.assertEqual(host_tls['dns_credential'], self.credential.name)
|
||||
|
||||
def test_workers_only_publish_exported_configuration(self):
|
||||
self.assertEqual(self.gateway_files()['applications.json'], {'entries': []})
|
||||
|
||||
export_caddy_config(self.export_dir)
|
||||
self.assertEqual([entry['id'] for entry in self.gateway_files()['applications.json']['entries']], ['node-app'])
|
||||
|
||||
self.node_app.upstream = 'http://10.0.0.21:8080'
|
||||
self.node_app.save()
|
||||
self.assertTrue(caddy_config_export.has_pending_export(self.export_dir))
|
||||
self.assertEqual(self.gateway_files()['applications.json']['entries'][0]['upstream'], 'http://10.0.0.20:8080')
|
||||
|
||||
export_caddy_config(self.export_dir)
|
||||
self.assertFalse(caddy_config_export.has_pending_export(self.export_dir))
|
||||
self.assertEqual(self.gateway_files()['applications.json']['entries'][0]['upstream'], 'http://10.0.0.21:8080')
|
||||
|
||||
def test_worker_receives_exported_certificate_without_dns_credentials(self):
|
||||
export_caddy_config(self.export_dir)
|
||||
worker_tls = self.gateway_files()['tls.json']
|
||||
self.assertEqual(worker_tls['dns_credentials'], {})
|
||||
self.assertEqual(worker_tls['hosts']['node.example.test']['certificate_file'], '')
|
||||
self.assertEqual(sync.build_section('certificates', self.worker).files, {})
|
||||
|
||||
# A renewed certificate reaches the worker without a new export
|
||||
self.write_exported_certificate('node.example.test')
|
||||
certificate_name = certificate_file_name(FAKE_CERTIFICATE)
|
||||
self.assertEqual(self.gateway_files()['tls.json']['hosts']['node.example.test']['certificate_file'], certificate_name)
|
||||
self.assertEqual(sync.build_section('certificates', self.worker).files, {
|
||||
f'{certificate_name}.crt': FAKE_CERTIFICATE,
|
||||
f'{certificate_name}.key': FAKE_PRIVATE_KEY,
|
||||
})
|
||||
|
||||
def test_new_worker_is_pending_until_exported(self):
|
||||
export_caddy_config(self.export_dir)
|
||||
self.assertFalse(caddy_config_export.has_pending_export(self.export_dir))
|
||||
Worker.objects.create(name='node-b')
|
||||
self.assertTrue(caddy_config_export.has_pending_export(self.export_dir))
|
||||
|
||||
|
||||
class ClusterTrafficAggregationTests(TestCase):
|
||||
def test_peer_traffic_is_summed_across_servers(self):
|
||||
worker = Worker.objects.create(name='node-a')
|
||||
WorkerStatus.objects.create(
|
||||
worker=worker, wireguard_status_updated=timezone.now(), wireguard_status={'wg0': {
|
||||
'PEER': {'transfer': {'tx': 5000, 'rx': 500}, 'latest-handshakes': '2000', 'endpoints': 'node:1'},
|
||||
'NODE_ONLY': {'transfer': {'tx': 7, 'rx': 3}, 'latest-handshakes': '5'},
|
||||
}},
|
||||
)
|
||||
WireguardStatusCache.objects.create(cache_type='master', processing_time_ms=0, data={'wg0': {
|
||||
'PEER': {'transfer': {'tx': 100, 'rx': 10}, 'latest-handshakes': '1000', 'endpoints': 'primary:1'},
|
||||
}})
|
||||
|
||||
func_concatenate_cluster_wireguard_status_cache()
|
||||
|
||||
combined = WireguardStatusCache.objects.filter(cache_type='cluster').latest('created').data['wg0']
|
||||
self.assertEqual(combined['PEER']['transfer'], {'tx': 5100, 'rx': 510})
|
||||
self.assertEqual(combined['PEER']['endpoints'], 'node:1')
|
||||
self.assertEqual(combined['PEER']['source_name'], 'worker_node-a')
|
||||
self.assertEqual(combined['NODE_ONLY']['transfer'], {'tx': 7, 'rx': 3})
|
||||
|
||||
+5
-7
@@ -1,12 +1,10 @@
|
||||
from django.urls import path
|
||||
|
||||
from cluster.cluster_api import api_cluster_status, api_get_worker_config_files, api_get_worker_dnsmasq_config, \
|
||||
api_worker_ping, api_submit_worker_wireguard_stats
|
||||
from cluster.cluster_api import api_legacy_cluster_status, api_manifest, api_report, api_section
|
||||
|
||||
urlpatterns = [
|
||||
path('status/', api_cluster_status, name='api_cluster_status'),
|
||||
path('worker/get_config_files/', api_get_worker_config_files, name='api_get_worker_config_files'),
|
||||
path('worker/get_dnsmasq_config/', api_get_worker_dnsmasq_config, name='api_get_worker_dnsmasq_config'),
|
||||
path('worker/ping/', api_worker_ping, name='api_worker_ping'),
|
||||
path('worker/submit_wireguard_stats/', api_submit_worker_wireguard_stats, name='api_submit_worker_wireguard_stats'),
|
||||
path('status/', api_legacy_cluster_status, name='api_legacy_cluster_status'),
|
||||
path('v2/manifest/', api_manifest, name='api_cluster_manifest'),
|
||||
path('v2/section/<str:section_name>/', api_section, name='api_cluster_section'),
|
||||
path('v2/report/', api_report, name='api_cluster_report'),
|
||||
]
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
from django.urls import path
|
||||
|
||||
from cluster.views import cluster_main, cluster_settings, worker_manage
|
||||
from cluster.views import cluster_main, cluster_settings, worker_details, worker_manage
|
||||
|
||||
urlpatterns = [
|
||||
path('', cluster_main, name='cluster_main'),
|
||||
path('worker/view/', worker_details, name='worker_details'),
|
||||
path('worker/manage/', worker_manage, name='worker_manage'),
|
||||
path('settings/', cluster_settings, name='cluster_settings'),
|
||||
]
|
||||
|
||||
+151
-80
@@ -2,33 +2,77 @@ from django.conf import settings
|
||||
from django.contrib import messages
|
||||
from django.contrib.auth.decorators import login_required
|
||||
from django.shortcuts import get_object_or_404, redirect, render
|
||||
from django.urls import reverse
|
||||
from django.utils.dateparse import parse_datetime
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
||||
from api.models import WireguardStatusCache
|
||||
from dns.functions import compress_dnsmasq_config
|
||||
from app_gateway.setup_defaults import get_webadmin_hostnames
|
||||
from user_manager.models import UserAcl
|
||||
from .forms import WorkerForm, ClusterSettingsForm
|
||||
from .models import ClusterSettings, Worker
|
||||
from .models import ClusterSettings, Worker, WorkerStatus
|
||||
from .sync import SECTION_NAMES, get_worker_sync_state, get_worker_warnings
|
||||
|
||||
|
||||
def get_worker_status(worker):
|
||||
try:
|
||||
return worker.workerstatus
|
||||
except WorkerStatus.DoesNotExist:
|
||||
return None
|
||||
|
||||
|
||||
def describe_worker(worker):
|
||||
"""Attach the synchronization state, warnings and reported data used by the cluster templates."""
|
||||
worker.sync_state = get_worker_sync_state(worker)
|
||||
worker.unsynced_sections = [
|
||||
section_name for section_name, section in worker.sync_state['sections'].items() if section['state'] != 'synced'
|
||||
]
|
||||
worker.error_messages = [
|
||||
f"{section_name}: {section['message']}" for section_name, section in worker.sync_state['sections'].items()
|
||||
if section['state'] == 'error' and section['message']
|
||||
]
|
||||
worker.warnings = get_worker_warnings(worker)
|
||||
# Reported data of disabled, refused or offline workers is only the last known state
|
||||
worker.operational = worker.enabled and not worker.error_status and worker.is_online
|
||||
worker_status = get_worker_status(worker)
|
||||
worker.status = worker_status
|
||||
worker.services = sorted((worker_status.services_status or {}).items()) if worker_status else []
|
||||
worker.runs_gateway = any(service == 'caddy' for service, state in worker.services)
|
||||
worker.outdated = bool(worker_status and worker_status.worker_version != settings.CLUSTER_WORKER_CURRENT_VERSION)
|
||||
return worker
|
||||
|
||||
|
||||
@login_required
|
||||
def cluster_main(request):
|
||||
"""Main cluster page with workers list"""
|
||||
if not UserAcl.objects.filter(user=request.user).filter(user_level__gte=50).exists():
|
||||
return render(request, 'access_denied.html', {'page_title': _('Access Denied')})
|
||||
|
||||
cluster_settings, created = ClusterSettings.objects.get_or_create(name='cluster_settings')
|
||||
page_title = _('Cluster')
|
||||
workers = Worker.objects.all().order_by('name')
|
||||
|
||||
master_cache_times = ", ".join([str(c.processing_time_ms) for c in WireguardStatusCache.objects.filter(cache_type='master').order_by('-created')])
|
||||
cluster_cache_times = ", ".join([str(c.processing_time_ms) for c in WireguardStatusCache.objects.filter(cache_type='cluster').order_by('-created')])
|
||||
cluster_settings, cluster_settings_created = ClusterSettings.objects.get_or_create(name='cluster_settings')
|
||||
workers = [describe_worker(worker) for worker in Worker.objects.all().select_related('workerstatus').order_by('name')]
|
||||
enabled_workers = [worker for worker in workers if worker.enabled]
|
||||
|
||||
master_cache_times = ", ".join(
|
||||
str(cache.processing_time_ms) for cache in WireguardStatusCache.objects.filter(cache_type='master').order_by('-created')
|
||||
)
|
||||
cluster_cache_times = ", ".join(
|
||||
str(cache.processing_time_ms) for cache in WireguardStatusCache.objects.filter(cache_type='cluster').order_by('-created')
|
||||
)
|
||||
|
||||
context = {
|
||||
'page_title': page_title,
|
||||
'page_title': _('Cluster'),
|
||||
'workers': workers,
|
||||
'current_worker_version': settings.CLUSTER_WORKER_CURRENT_VERSION,
|
||||
'cluster_settings': cluster_settings,
|
||||
'current_worker_version': settings.CLUSTER_WORKER_CURRENT_VERSION,
|
||||
'summary': {
|
||||
'enabled': len(enabled_workers),
|
||||
'online': sum(1 for worker in enabled_workers if worker.operational),
|
||||
'synced': sum(1 for worker in enabled_workers if worker.operational and worker.sync_state['state'] == 'synced'),
|
||||
'attention': sum(
|
||||
1 for worker in enabled_workers
|
||||
if worker.sync_state['state'] == 'error' or worker.error_status or worker.warnings['stopped_services']
|
||||
or worker.warnings['certificate_issues'] or not worker.is_online
|
||||
),
|
||||
},
|
||||
'cache_refresh_interval': settings.WIREGUARD_STATUS_CACHE_REFRESH_INTERVAL,
|
||||
'cache_enabled': settings.WIREGUARD_STATUS_CACHE_ENABLED,
|
||||
'cache_web_load_previous_count': settings.WIREGUARD_STATUS_CACHE_WEB_LOAD_PREVIOUS_COUNT,
|
||||
@@ -39,119 +83,146 @@ def cluster_main(request):
|
||||
|
||||
|
||||
@login_required
|
||||
def worker_manage(request):
|
||||
"""Add/Edit worker view"""
|
||||
def worker_details(request):
|
||||
if not UserAcl.objects.filter(user=request.user).filter(user_level__gte=50).exists():
|
||||
return render(request, 'access_denied.html', {'page_title': _('Access Denied')})
|
||||
|
||||
|
||||
worker = describe_worker(get_object_or_404(Worker.objects.select_related('workerstatus'), uuid=request.GET.get('uuid')))
|
||||
cluster_settings, cluster_settings_created = ClusterSettings.objects.get_or_create(name='cluster_settings')
|
||||
|
||||
sections = []
|
||||
for section_name in SECTION_NAMES:
|
||||
section_state = worker.sync_state['sections'].get(section_name, {})
|
||||
reported = (worker.status.section_status or {}).get(section_name, {}) if worker.status else {}
|
||||
sections.append({
|
||||
'name': section_name,
|
||||
'state': section_state.get('state', 'pending'),
|
||||
'message': section_state.get('message', ''),
|
||||
'updated': parse_datetime(reported.get('updated', '')) if reported.get('updated') else None,
|
||||
})
|
||||
|
||||
certificates = []
|
||||
if worker.status:
|
||||
for hostname, certificate in sorted((worker.status.certificate_status or {}).items()):
|
||||
certificates.append({
|
||||
'hostname': hostname,
|
||||
'status': certificate.get('status', ''),
|
||||
'issuer': certificate.get('issuer', ''),
|
||||
'not_after': parse_datetime(certificate['not_after']) if certificate.get('not_after') else None,
|
||||
})
|
||||
|
||||
webadmin_hostnames = get_webadmin_hostnames()
|
||||
primary_address = f"https://{webadmin_hostnames[0] if webadmin_hostnames else request.get_host()}"
|
||||
|
||||
context = {
|
||||
'page_title': worker.name,
|
||||
'worker': worker,
|
||||
'cluster_settings': cluster_settings,
|
||||
'current_worker_version': settings.CLUSTER_WORKER_CURRENT_VERSION,
|
||||
'sections': sections,
|
||||
'certificates': certificates,
|
||||
'primary_address': primary_address,
|
||||
'time_zone': settings.TIME_ZONE,
|
||||
}
|
||||
return render(request, 'cluster/worker_details.html', context)
|
||||
|
||||
|
||||
@login_required
|
||||
def worker_manage(request):
|
||||
if not UserAcl.objects.filter(user=request.user).filter(user_level__gte=50).exists():
|
||||
return render(request, 'access_denied.html', {'page_title': _('Access Denied')})
|
||||
|
||||
worker = None
|
||||
if 'uuid' in request.GET:
|
||||
worker = get_object_or_404(Worker, uuid=request.GET['uuid'])
|
||||
form = WorkerForm(instance=worker)
|
||||
page_title = _('Edit Worker: ') + worker.name
|
||||
|
||||
|
||||
if request.GET.get('action') == 'delete':
|
||||
worker_name = worker.name
|
||||
if request.GET.get('confirmation') == 'delete':
|
||||
worker.delete()
|
||||
messages.success(request, _('Worker deleted|Worker deleted: ') + worker_name)
|
||||
return redirect('/cluster/')
|
||||
else:
|
||||
messages.warning(request, _('Worker not deleted|Invalid confirmation.'))
|
||||
return redirect('/cluster/')
|
||||
return redirect(reverse('cluster_main'))
|
||||
else:
|
||||
form = WorkerForm()
|
||||
page_title = _('Add Worker')
|
||||
|
||||
if request.method == 'POST':
|
||||
if worker:
|
||||
form = WorkerForm(request.POST, instance=worker)
|
||||
form = WorkerForm(request.POST or None, instance=worker)
|
||||
if form.is_valid():
|
||||
is_new_worker = worker is None
|
||||
worker = form.save()
|
||||
if is_new_worker:
|
||||
messages.success(request, _('Worker created|Worker created: ') + worker.name)
|
||||
else:
|
||||
form = WorkerForm(request.POST)
|
||||
|
||||
if form.is_valid():
|
||||
worker = form.save()
|
||||
if worker.pk:
|
||||
messages.success(request, _('Worker updated|Worker updated: ') + worker.name)
|
||||
else:
|
||||
messages.success(request, _('Worker created|Worker created: ') + worker.name)
|
||||
return redirect('/cluster/')
|
||||
messages.success(request, _('Worker updated|Worker updated: ') + worker.name)
|
||||
return redirect(reverse('worker_details') + f'?uuid={worker.uuid}')
|
||||
|
||||
form_description = {
|
||||
'size': 'col-lg-6',
|
||||
'content': _('''
|
||||
<h5>Worker Configuration</h5>
|
||||
<p>Configure a cluster worker node that will synchronize with this primary instance.</p>
|
||||
|
||||
<h5>Name</h5>
|
||||
<p>A unique name to identify this worker.</p>
|
||||
|
||||
<h5>IP Address</h5>
|
||||
<p>The IP address of the worker node. Leave empty if IP lock is disabled.</p>
|
||||
|
||||
<h5>IP Lock</h5>
|
||||
<p>When enabled, the worker can only connect from the specified IP address.</p>
|
||||
|
||||
<h5>Location Information</h5>
|
||||
<p>Optional location details for this worker (country, city, hostname).</p>
|
||||
<h5>Worker</h5>
|
||||
<p>A worker is a cluster node that replicates WireGuard, firewall and DNS from this primary server and, optionally, publishes App Gateway applications.</p>
|
||||
|
||||
<h5>Name and location</h5>
|
||||
<p>Used to identify the worker. The <strong>hostname</strong> is the address clients use to connect to this worker: it is written as the endpoint of the peer configurations downloaded for this server.</p>
|
||||
|
||||
<h5>Token</h5>
|
||||
<p>Secret used by the worker to authenticate with this primary server. Changing it disconnects the worker until its configuration is updated.</p>
|
||||
|
||||
<h5>IP lock</h5>
|
||||
<p>When enabled, the worker is only accepted from the IP address below. Without it, the address is updated automatically from the last connection.</p>
|
||||
|
||||
<h5>Enabled</h5>
|
||||
<p>Disabled workers are refused and shut down their WireGuard interfaces until enabled again.</p>
|
||||
''')
|
||||
}
|
||||
|
||||
|
||||
context = {
|
||||
'page_title': page_title,
|
||||
'form': form,
|
||||
'worker': worker,
|
||||
'page_title': page_title,
|
||||
'form': form,
|
||||
'worker': worker,
|
||||
'instance': worker,
|
||||
'form_description': form_description
|
||||
'form_description': form_description,
|
||||
}
|
||||
return render(request, 'generic_form.html', context)
|
||||
|
||||
|
||||
@login_required
|
||||
def cluster_settings(request):
|
||||
"""Cluster settings configuration"""
|
||||
if not UserAcl.objects.filter(user=request.user).filter(user_level__gte=50).exists():
|
||||
return render(request, 'access_denied.html', {'page_title': _('Access Denied')})
|
||||
|
||||
cluster_settings, created = ClusterSettings.objects.get_or_create(name='cluster_settings')
|
||||
page_title = _('Cluster Settings')
|
||||
|
||||
if request.method == 'POST':
|
||||
form = ClusterSettingsForm(request.POST, instance=cluster_settings)
|
||||
if form.is_valid():
|
||||
form.save()
|
||||
messages.success(request, _('Cluster settings updated successfully.'))
|
||||
if cluster_settings.enabled:
|
||||
if cluster_settings.config_version == 0:
|
||||
cluster_settings.config_version += 1
|
||||
cluster_settings.save()
|
||||
compress_dnsmasq_config()
|
||||
return redirect('/cluster/')
|
||||
else:
|
||||
form = ClusterSettingsForm(instance=cluster_settings)
|
||||
|
||||
cluster_settings, cluster_settings_created = ClusterSettings.objects.get_or_create(name='cluster_settings')
|
||||
|
||||
form = ClusterSettingsForm(request.POST or None, instance=cluster_settings)
|
||||
if form.is_valid():
|
||||
form.save()
|
||||
messages.success(request, _('Cluster settings updated successfully.'))
|
||||
return redirect(reverse('cluster_main'))
|
||||
|
||||
form_description = {
|
||||
'size': 'col-lg-6',
|
||||
'content': _('''
|
||||
<h5>Cluster Enabled</h5>
|
||||
<p>When disabled, workers are refused and shut down their WireGuard interfaces, and App Gateway applications are only published by this primary server. Requires the WireGuard status cache.</p>
|
||||
|
||||
<h5>Primary Enable WireGuard</h5>
|
||||
<p>Lists this primary server as an option when downloading peer configurations and VPN invites.</p>
|
||||
|
||||
<h5>Cluster Mode</h5>
|
||||
<p>Configure how the cluster operates and synchronizes configurations between nodes.</p>
|
||||
|
||||
<h5>Sync Intervals</h5>
|
||||
<p>Configure how frequently statistics and cache data are synchronized between cluster nodes.</p>
|
||||
|
||||
<h5>Restart Mode</h5>
|
||||
<p>Choose whether WireGuard services should be automatically restarted when configurations change, or if manual intervention is required.</p>
|
||||
|
||||
<p><strong>Mirror</strong>: every worker receives the same WireGuard instances, peers, firewall and DNS configuration. Peers can connect to any server.</p>
|
||||
|
||||
<h5>Worker Display</h5>
|
||||
<p>Select how workers should be identified in the interface - by name, server address, location, or a combination.</p>
|
||||
<p>How workers are named in server selections (peer configuration downloads and VPN invites).</p>
|
||||
''')
|
||||
}
|
||||
|
||||
|
||||
context = {
|
||||
'page_title': page_title,
|
||||
'page_title': _('Cluster Settings'),
|
||||
'form': form,
|
||||
'cluster_settings': cluster_settings,
|
||||
'instance': cluster_settings,
|
||||
'form_description': form_description
|
||||
'form_description': form_description,
|
||||
}
|
||||
return render(request, 'generic_form.html', context)
|
||||
|
||||
Reference in New Issue
Block a user