Rework cluster into v2: section-based worker sync and App Gateway on workers

This commit is contained in:
Eduardo Silva
2026-11-20 11:56:30 -03:00
parent dee270695d
commit 7468cdf623
41 changed files with 2766 additions and 1014 deletions
+132 -182
View File
@@ -1,220 +1,170 @@
import glob
import json
import os
from django.conf import settings
from django.http import JsonResponse, FileResponse
from django.core.exceptions import ValidationError
from django.http import JsonResponse
from django.utils import timezone
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.gzip import gzip_page
from django.views.decorators.http import require_GET, require_POST
from .models import ClusterSettings, Worker, WorkerStatus
from cluster.models import ClusterSettings, Worker, WorkerStatus
from cluster.sync import SECTION_NAMES, build_manifest, build_section
REPORT_STATUS_FIELDS = ('status', 'message')
def get_ip_address(request):
ip_address = request.META.get('HTTP_X_FORWARDED_FOR')
if ip_address:
ip_address = ip_address.split(',')[0]
ip_address = ip_address.split(',')[0].strip()
else:
ip_address = request.META.get('REMOTE_ADDR')
return ip_address
def get_cluster_settings():
cluster_settings, created = ClusterSettings.objects.get_or_create(name='cluster_settings')
return {
'enabled': cluster_settings.enabled,
'primary_enable_wireguard': cluster_settings.primary_enable_wireguard,
'stats_sync_interval': settings.WIREGUARD_STATUS_CACHE_REFRESH_INTERVAL,
'cluster_mode': cluster_settings.cluster_mode,
'restart_mode': cluster_settings.restart_mode,
'config_version': cluster_settings.config_version,
'dns_version': cluster_settings.dns_version,
}
def _error_response(message, status):
return JsonResponse({'status': 'error', 'message': message}, status=status)
def get_worker(request):
success = True
ip_address = get_ip_address(request)
token = request.GET.get('token', '')
"""
Authenticate the worker from the Authorization header and update its status.
Returns (worker, None) on success or (None, JsonResponse) when the request must be rejected.
"""
authorization = request.headers.get('Authorization', '')
if not authorization.startswith('Bearer '):
return None, _error_response('Worker not found', 403)
try:
worker = Worker.objects.get(token=token)
except:
return None, False
worker = Worker.objects.get(token=authorization[len('Bearer '):].strip())
except (Worker.DoesNotExist, ValidationError, ValueError):
return None, _error_response('Worker not found', 403)
worker_status, created = WorkerStatus.objects.get_or_create(worker=worker)
worker_status, worker_status_created = WorkerStatus.objects.get_or_create(worker=worker)
try:
worker_config_version = int(request.GET.get('worker_config_version'))
worker_version = int(request.GET.get('worker_version'))
worker_dns_version = int(request.GET.get('worker_dns_version'))
except:
worker.error_status = 'missing_version'
worker.save()
return worker, False
if worker.error_status == 'missing_version':
worker.error_status = ''
worker.save()
if worker_version < settings.CLUSTER_WORKER_MINIMUM_VERSION:
worker.error_status = 'update_required'
worker.save()
return worker, False
if worker.error_status == 'update_required':
worker.error_status = ''
worker.save()
if worker_status.config_version != worker_config_version:
worker_status.config_version = worker_config_version
if worker_status.worker_version != worker_version:
worker_status.worker_version = worker_version
if worker_status.dns_version != worker_dns_version:
worker_status.dns_version = worker_dns_version
worker_version = int(request.headers.get('X-Worker-Version', '0'))
except ValueError:
worker_version = 0
worker_status.worker_version = worker_version
worker_status.last_seen = timezone.now()
worker_status.save()
if not worker.ip_address:
worker_status.save(update_fields=['worker_version', 'last_seen', 'updated'])
ip_address = get_ip_address(request)
error_status = ''
if worker_version < settings.CLUSTER_WORKER_MINIMUM_VERSION:
error_status = 'update_required'
elif worker.ip_lock and worker.ip_address and worker.ip_address != ip_address:
error_status = 'ip_lock'
elif not worker.enabled:
error_status = 'worker_disabled'
else:
cluster_settings, cluster_settings_created = ClusterSettings.objects.get_or_create(name='cluster_settings')
if cluster_settings.enabled and not settings.WIREGUARD_STATUS_CACHE_ENABLED:
cluster_settings.enabled = False
cluster_settings.save()
if not cluster_settings.enabled:
error_status = 'cluster_disabled'
update_fields = []
if not worker.ip_lock and worker.ip_address != ip_address:
worker.ip_address = ip_address
worker.save()
update_fields.append('ip_address')
if worker.error_status != error_status:
worker.error_status = error_status
update_fields.append('error_status')
if update_fields:
worker.save(update_fields=update_fields + ['updated'])
if worker.ip_lock:
if worker.ip_address == ip_address:
if worker.error_status == 'ip_lock':
worker.error_status = ''
worker.save()
else:
worker.error_status = 'ip_lock'
worker.save()
success = False
else:
if worker.ip_address != ip_address:
worker.ip_address = ip_address
worker.save()
if worker.enabled:
if worker.error_status == 'worker_disabled':
worker.error_status = ''
worker.save()
else:
worker.error_status = 'worker_disabled'
worker.save()
success = False
cluster_settings, created = ClusterSettings.objects.get_or_create(name='cluster_settings')
if not settings.WIREGUARD_STATUS_CACHE_ENABLED and cluster_settings.enabled:
cluster_settings.enabled = False
cluster_settings.save()
if cluster_settings.enabled:
if worker.error_status == 'cluster_disabled':
worker.error_status = ''
worker.save()
else:
worker.error_status = 'cluster_disabled'
worker.save()
success = False
return worker, success
if error_status:
return None, _error_response(error_status, 400)
return worker, None
def api_get_worker_dnsmasq_config(request):
dnsmasq_file = "/etc/dnsmasq/dnsmasq_config.tar.gz"
worker, success = get_worker(request)
if worker:
if worker.error_status or not success:
data = {'status': 'error', 'message': worker.error_status}
return JsonResponse(data, status=400)
else:
data = {'status': 'error', 'message': 'Worker not found'}
return JsonResponse(data, status=403)
def api_legacy_cluster_status(request):
# Workers before version 12 poll this endpoint. A 403 makes them shut down WireGuard and stop polling.
return JsonResponse({
'status': 'error',
'message': 'update_required',
'minimum_worker_version': settings.CLUSTER_WORKER_MINIMUM_VERSION,
}, status=403)
if not os.path.exists(dnsmasq_file):
data = {'status': 'error', 'message': 'dnsmasq configuration not found'}
return JsonResponse(data, status=404)
response = FileResponse(open(dnsmasq_file, "rb"), content_type="application/gzip")
response["Content-Disposition"] = 'attachment; filename="dnsmasq_config.tar.gz"'
response["Content-Length"] = str(os.path.getsize(dnsmasq_file))
return response
@require_GET
def api_manifest(request):
worker, error_response = get_worker(request)
if error_response:
return error_response
return JsonResponse({
'status': 'success',
'worker_minimum_version': settings.CLUSTER_WORKER_MINIMUM_VERSION,
'sections': build_manifest(worker),
})
@gzip_page
@require_GET
def api_section(request, section_name):
worker, error_response = get_worker(request)
if error_response:
return error_response
if section_name not in SECTION_NAMES:
return _error_response('Section not found', 404)
return JsonResponse({'status': 'success', 'section': section_name, **build_section(section_name, worker).as_dict()})
def _clean_section_report(sections):
cleaned_sections = {}
for section_name, report in sections.items():
if section_name not in SECTION_NAMES or not isinstance(report, dict):
continue
status = report.get('status')
if status not in ('ok', 'error'):
continue
cleaned_sections[section_name] = {
'hash': str(report.get('hash', ''))[:64],
'status': status,
'message': str(report.get('message', ''))[:1000],
'updated': timezone.now().isoformat(),
}
return cleaned_sections
@csrf_exempt
def api_submit_worker_wireguard_stats(request):
worker, success = get_worker(request)
if worker:
if worker.error_status or not success:
data = {'status': 'error', 'message': worker.error_status}
return JsonResponse(data, status=400)
else:
data = {'status': 'error', 'message': 'Worker not found'}
return JsonResponse(data, status=403)
@require_POST
def api_report(request):
worker, error_response = get_worker(request)
if error_response:
return error_response
try:
if request.method == 'POST':
payload = json.loads(request.body)
worker_status = worker.workerstatus
worker_status.wireguard_status = payload
worker_status.wireguard_status_updated = timezone.now()
worker_status.save()
data = {'status': 'success', 'message': 'Stats received'}
return JsonResponse(data, status=200)
except Exception as e:
pass
payload = json.loads(request.body)
except (ValueError, UnicodeDecodeError):
return _error_response('Invalid JSON', 400)
if not isinstance(payload, dict):
return _error_response('Invalid JSON', 400)
data = {'status': 'error', 'message': 'Stats not received'}
return JsonResponse(data, status=400)
worker_status = worker.workerstatus
update_fields = []
if isinstance(payload.get('sections'), dict):
section_status = dict(worker_status.section_status or {})
section_status.update(_clean_section_report(payload['sections']))
worker_status.section_status = section_status
update_fields.append('section_status')
def api_get_worker_config_files(request):
worker, success = get_worker(request)
if worker:
if worker.error_status or not success:
data = {'status': 'error', 'message': worker.error_status}
return JsonResponse(data, status=400)
else:
data = {'status': 'error', 'message': 'Worker not found'}
return JsonResponse(data, status=403)
if isinstance(payload.get('services'), dict):
worker_status.services_status = payload['services']
update_fields.append('services_status')
config_files = (
glob.glob('/etc/wireguard/wg*.conf') +
glob.glob('/etc/wireguard/wg-firewall.sh')
)
if isinstance(payload.get('certificates'), dict):
worker_status.certificate_status = payload['certificates']
worker_status.certificate_status_updated = timezone.now()
update_fields += ['certificate_status', 'certificate_status_updated']
files = {}
if isinstance(payload.get('wireguard_status'), dict):
worker_status.wireguard_status = payload['wireguard_status']
worker_status.wireguard_status_updated = timezone.now()
update_fields += ['wireguard_status', 'wireguard_status_updated']
for path in config_files:
filename = os.path.basename(path)
with open(path, 'r') as f:
files[filename] = f.read()
data = {'status': 'success', 'files': files, 'cluster_settings': get_cluster_settings()}
return JsonResponse(data, status=200)
def api_worker_ping(request):
worker, success = get_worker(request)
if worker:
if worker.error_status or not success:
data = {'status': 'error', 'message': worker.error_status}
return JsonResponse(data, status=400)
else:
data = {'status': 'error', 'message': 'Worker not found'}
return JsonResponse(data, status=403)
data = {
'status': 'success',
'worker_error_status': worker.error_status,
}
return JsonResponse(data, status=200)
def api_cluster_status(request):
worker, success = get_worker(request)
if worker:
if worker.error_status or not success:
data = {'status': 'error', 'message': worker.error_status}
return JsonResponse(data, status=400)
else:
data = {'status': 'error', 'message': 'Worker not found'}
return JsonResponse(data, status=403)
data = {'status': 'success', 'worker_error_status': worker.error_status, 'cluster_settings': get_cluster_settings()}
return JsonResponse(data, status=200)
if update_fields:
worker_status.save(update_fields=update_fields + ['updated'])
return JsonResponse({'status': 'success'})