Files
wireguard_webadmin/cluster/tests.py
T

274 lines
15 KiB
Python
Raw Normal View History

import json
import os
import tempfile
from unittest.mock import patch
from django.test import TestCase, override_settings
from django.utils import timezone
from api.models import WireguardStatusCache
from api.views import func_concatenate_cluster_wireguard_status_cache
from app_gateway import caddy_config_export
from app_gateway.caddy_config_export import certificate_file_name, export_caddy_config
from app_gateway.models import Application, ApplicationHost, DnsChallengeCredential
from cluster import sync
from cluster.models import ClusterSettings, Worker, WorkerStatus
from dns.models import DNSSettings
from firewall.tools import generate_firewall_script
FAKE_CERTIFICATE = '-----BEGIN CERTIFICATE-----\nfake\n-----END CERTIFICATE-----\n'
FAKE_PRIVATE_KEY = '-----BEGIN PRIVATE KEY-----\nfake\n-----END PRIVATE KEY-----\n'
class ClusterTestMixin:
"""Isolated directories for the files read and written by the cluster sync and the Caddy export."""
def setUp(self):
super().setUp()
self.temp_dir = tempfile.TemporaryDirectory()
self.addCleanup(self.temp_dir.cleanup)
self.wireguard_dir = os.path.join(self.temp_dir.name, 'wireguard')
self.dnsmasq_dir = os.path.join(self.temp_dir.name, 'dnsmasq')
self.export_dir = os.path.join(self.temp_dir.name, 'caddy_json_export')
self.certificate_export_dir = os.path.join(self.temp_dir.name, 'caddy_cert_export')
for directory in (self.wireguard_dir, self.dnsmasq_dir, self.export_dir, self.certificate_export_dir):
os.makedirs(directory)
with open(os.path.join(self.wireguard_dir, 'wg0.conf'), 'w', encoding='utf-8') as config_file:
config_file.write('[Interface]\nListenPort = 51820\n')
for target, value in (
('cluster.sync.WIREGUARD_CONFIG_DIR', self.wireguard_dir),
('cluster.sync.DNSMASQ_CONFIG_DIR', self.dnsmasq_dir),
('app_gateway.caddy_config_export.CADDY_EXPORT_DIR', self.export_dir),
('app_gateway.caddy_config_export.CERTIFICATE_EXPORT_DIR', self.certificate_export_dir),
):
patcher = patch(target, value)
patcher.start()
self.addCleanup(patcher.stop)
DNSSettings.objects.get_or_create(name='dns_settings')
ClusterSettings.objects.update_or_create(name='cluster_settings', defaults={'enabled': True})
self.worker = Worker.objects.create(name='node-a', hostname='node-a.example.test')
self.headers = {'HTTP_AUTHORIZATION': f'Bearer {self.worker.token}', 'HTTP_X_WORKER_VERSION': '12'}
def write_exported_certificate(self, hostname):
host_dir = os.path.join(self.certificate_export_dir, hostname)
os.makedirs(host_dir, exist_ok=True)
with open(os.path.join(host_dir, 'certificate.crt'), 'w', encoding='utf-8') as certificate_file:
certificate_file.write(FAKE_CERTIFICATE)
with open(os.path.join(host_dir, 'certificate.key'), 'w', encoding='utf-8') as key_file:
key_file.write(FAKE_PRIVATE_KEY)
@override_settings(CLUSTER_WORKER_MINIMUM_VERSION=12, WIREGUARD_STATUS_CACHE_ENABLED=True)
class ClusterApiTests(ClusterTestMixin, TestCase):
def test_legacy_status_endpoint_stops_old_workers_without_token(self):
response = self.client.get('/api/cluster/status/')
self.assertEqual(response.status_code, 403)
self.assertEqual(response.json()['message'], 'update_required')
self.assertEqual(response.json()['minimum_worker_version'], 12)
def test_manifest_requires_token(self):
self.assertEqual(self.client.get('/api/cluster/v2/manifest/').status_code, 403)
response = self.client.get('/api/cluster/v2/manifest/', HTTP_AUTHORIZATION='Bearer not-a-uuid')
self.assertEqual(response.status_code, 403)
def test_worker_is_refused_with_reason(self):
scenarios = (
({'HTTP_X_WORKER_VERSION': '11'}, {}, 'update_required'),
({}, {'enabled': False}, 'worker_disabled'),
({}, {'ip_lock': True, 'ip_address': '203.0.113.10'}, 'ip_lock'),
)
for header_override, worker_fields, expected_error in scenarios:
with self.subTest(expected_error=expected_error):
Worker.objects.filter(pk=self.worker.pk).update(**{'enabled': True, 'ip_lock': False, **worker_fields})
response = self.client.get('/api/cluster/v2/manifest/', **{**self.headers, **header_override})
self.assertEqual(response.status_code, 400)
self.assertEqual(response.json()['message'], expected_error)
self.worker.refresh_from_db()
self.assertEqual(self.worker.error_status, expected_error)
def test_disabled_cluster_refuses_workers(self):
ClusterSettings.objects.filter(name='cluster_settings').update(enabled=False)
response = self.client.get('/api/cluster/v2/manifest/', **self.headers)
self.assertEqual(response.status_code, 400)
self.assertEqual(response.json()['message'], 'cluster_disabled')
def test_manifest_hashes_match_sections(self):
response = self.client.get('/api/cluster/v2/manifest/', **self.headers)
self.assertEqual(response.status_code, 200)
manifest = response.json()['sections']
self.assertEqual(list(manifest), list(sync.SECTION_NAMES))
for section_name, section_hash in manifest.items():
with self.subTest(section=section_name):
section_response = self.client.get(f'/api/cluster/v2/section/{section_name}/', **self.headers)
self.assertEqual(section_response.status_code, 200)
self.assertEqual(section_response.json()['hash'], section_hash)
self.assertEqual(self.client.get('/api/cluster/v2/section/unknown/', **self.headers).status_code, 404)
def test_report_stores_worker_status(self):
report = {
'sections': {
'dns': {'hash': 'abc', 'status': 'error', 'message': 'dnsmasq failed'},
'unknown': {'hash': 'abc', 'status': 'ok'},
'settings': {'hash': 'def', 'status': 'invalid'},
},
'services': {'dnsmasq': 'running'},
'certificates': {'app.example.test': {'status': 'trusted'}},
'wireguard_status': {'wg0': {}},
}
response = self.client.post(
'/api/cluster/v2/report/', data=json.dumps(report), content_type='application/json', **self.headers
)
self.assertEqual(response.status_code, 200)
worker_status = WorkerStatus.objects.get(worker=self.worker)
self.assertEqual(list(worker_status.section_status), ['dns'])
self.assertEqual(worker_status.section_status['dns']['message'], 'dnsmasq failed')
self.assertEqual(worker_status.services_status, {'dnsmasq': 'running'})
self.assertEqual(worker_status.certificate_status, report['certificates'])
self.assertEqual(worker_status.wireguard_status, {'wg0': {}})
self.assertIsNotNone(worker_status.certificate_status_updated)
self.assertIsNotNone(worker_status.wireguard_status_updated)
class ClusterSyncTests(ClusterTestMixin, TestCase):
def report_sections(self, manifest, overrides=None):
section_status = {name: {'hash': section_hash, 'status': 'ok'} for name, section_hash in manifest.items()}
section_status.update(overrides or {})
WorkerStatus.objects.update_or_create(worker=self.worker, defaults={'section_status': section_status})
self.worker.refresh_from_db()
def test_sync_state(self):
manifest = sync.build_manifest(self.worker)
self.assertEqual(sync.get_worker_sync_state(self.worker, manifest)['state'], 'pending')
self.report_sections(manifest)
self.assertEqual(sync.get_worker_sync_state(self.worker, manifest)['state'], 'synced')
self.report_sections(manifest, {'dns': {'hash': manifest['dns'], 'status': 'error', 'message': 'failed'}})
sync_state = sync.get_worker_sync_state(self.worker, manifest)
self.assertEqual(sync_state['state'], 'error')
self.assertEqual(sync_state['sections']['dns']['message'], 'failed')
# An error reported for older content means the node still has to try the new one
self.report_sections(manifest, {'dns': {'hash': 'old', 'status': 'error', 'message': 'failed'}})
self.assertEqual(sync.get_worker_sync_state(self.worker, manifest)['state'], 'pending')
def test_wireguard_section_follows_config_files(self):
first_hash = sync.build_section('wireguard', self.worker).hash
self.assertEqual(sync.build_section('wireguard', self.worker).hash, first_hash)
with open(os.path.join(self.wireguard_dir, 'wg1.conf'), 'w', encoding='utf-8') as config_file:
config_file.write('[Interface]\nListenPort = 51821\n')
section = sync.build_section('wireguard', self.worker)
self.assertNotEqual(section.hash, first_hash)
self.assertEqual(sorted(section.files), ['wg0.conf', 'wg1.conf'])
def test_dns_section_uses_node_configuration(self):
section = sync.build_section('dns', self.worker)
main_config = section.files[sync.DNSMASQ_MAIN_CONFIG]
self.assertIn('bind-dynamic', main_config)
self.assertIn('interface=wg*', main_config)
self.assertNotIn('listen-address=0.0.0.0', main_config)
def test_firewall_section_is_stable_and_node_specific(self):
self.assertEqual(sync.build_section('firewall', self.worker).hash, sync.build_section('firewall', self.worker).hash)
node_script = generate_firewall_script(cluster_node=True)
primary_script = generate_firewall_script()
self.assertNotIn('DNS_IP', node_script)
self.assertNotIn('--dport 8000', node_script)
self.assertNotIn('generated by', node_script)
self.assertIn('DNS_IP', primary_script)
class ClusterGatewaySyncTests(ClusterTestMixin, TestCase):
def setUp(self):
super().setUp()
self.credential = DnsChallengeCredential.objects.create(
display_name='Cloudflare', provider='cloudflare', credentials={'api_token': 'x' * 40}
)
self.node_app = Application.objects.create(
display_name='Node App', upstream='http://10.0.0.20:8080', serve_on_primary=False
)
self.node_app.workers.add(self.worker)
ApplicationHost.objects.create(
application=self.node_app, hostname='node.example.test', certificate_type='dns_challenge',
dns_credential=self.credential,
)
def gateway_files(self):
return {
filename: json.loads(content)
for filename, content in sync.build_section('gateway', self.worker).files.items()
}
def test_primary_manages_certificates_of_worker_only_applications(self):
export_caddy_config(self.export_dir)
with open(os.path.join(self.export_dir, 'applications.json'), encoding='utf-8') as applications_file:
primary_applications = json.load(applications_file)
with open(os.path.join(self.export_dir, 'tls.json'), encoding='utf-8') as tls_file:
primary_tls = json.load(tls_file)
self.assertNotIn('node-app', [entry['id'] for entry in primary_applications['entries']])
host_tls = primary_tls['hosts']['node.example.test']
self.assertTrue(host_tls['certificate_only'])
self.assertTrue(host_tls['export_certificate'])
self.assertEqual(host_tls['dns_credential'], self.credential.name)
def test_workers_only_publish_exported_configuration(self):
self.assertEqual(self.gateway_files()['applications.json'], {'entries': []})
export_caddy_config(self.export_dir)
self.assertEqual([entry['id'] for entry in self.gateway_files()['applications.json']['entries']], ['node-app'])
self.node_app.upstream = 'http://10.0.0.21:8080'
self.node_app.save()
self.assertTrue(caddy_config_export.has_pending_export(self.export_dir))
self.assertEqual(self.gateway_files()['applications.json']['entries'][0]['upstream'], 'http://10.0.0.20:8080')
export_caddy_config(self.export_dir)
self.assertFalse(caddy_config_export.has_pending_export(self.export_dir))
self.assertEqual(self.gateway_files()['applications.json']['entries'][0]['upstream'], 'http://10.0.0.21:8080')
def test_worker_receives_exported_certificate_without_dns_credentials(self):
export_caddy_config(self.export_dir)
worker_tls = self.gateway_files()['tls.json']
self.assertEqual(worker_tls['dns_credentials'], {})
self.assertEqual(worker_tls['hosts']['node.example.test']['certificate_file'], '')
self.assertEqual(sync.build_section('certificates', self.worker).files, {})
# A renewed certificate reaches the worker without a new export
self.write_exported_certificate('node.example.test')
certificate_name = certificate_file_name(FAKE_CERTIFICATE)
self.assertEqual(self.gateway_files()['tls.json']['hosts']['node.example.test']['certificate_file'], certificate_name)
self.assertEqual(sync.build_section('certificates', self.worker).files, {
f'{certificate_name}.crt': FAKE_CERTIFICATE,
f'{certificate_name}.key': FAKE_PRIVATE_KEY,
})
def test_new_worker_is_pending_until_exported(self):
export_caddy_config(self.export_dir)
self.assertFalse(caddy_config_export.has_pending_export(self.export_dir))
Worker.objects.create(name='node-b')
self.assertTrue(caddy_config_export.has_pending_export(self.export_dir))
class ClusterTrafficAggregationTests(TestCase):
def test_peer_traffic_is_summed_across_servers(self):
worker = Worker.objects.create(name='node-a')
WorkerStatus.objects.create(
worker=worker, wireguard_status_updated=timezone.now(), wireguard_status={'wg0': {
'PEER': {'transfer': {'tx': 5000, 'rx': 500}, 'latest-handshakes': '2000', 'endpoints': 'node:1'},
'NODE_ONLY': {'transfer': {'tx': 7, 'rx': 3}, 'latest-handshakes': '5'},
}},
)
WireguardStatusCache.objects.create(cache_type='master', processing_time_ms=0, data={'wg0': {
'PEER': {'transfer': {'tx': 100, 'rx': 10}, 'latest-handshakes': '1000', 'endpoints': 'primary:1'},
}})
func_concatenate_cluster_wireguard_status_cache()
combined = WireguardStatusCache.objects.filter(cache_type='cluster').latest('created').data['wg0']
self.assertEqual(combined['PEER']['transfer'], {'tx': 5100, 'rx': 510})
self.assertEqual(combined['PEER']['endpoints'], 'node:1')
self.assertEqual(combined['PEER']['source_name'], 'worker_node-a')
self.assertEqual(combined['NODE_ONLY']['transfer'], {'tx': 7, 'rx': 3})