Add certificate retry, optional fallback and DNS propagation fix to App Gateway hosts

This commit is contained in:
Eduardo Silva
2026-11-18 17:33:44 -03:00
parent be7965512b
commit 77e046dc1f
11 changed files with 260 additions and 33 deletions
+5 -1
View File
@@ -134,7 +134,11 @@ def build_tls_data():
hosts = {}
dns_credentials = {}
for application_host in ApplicationHost.objects.select_related('dns_credential'):
host_entry = {'certificate_type': application_host.certificate_type}
host_entry = {
'certificate_type': application_host.certificate_type,
'certificate_fallback': application_host.certificate_fallback,
'certificate_generation': application_host.certificate_generation,
}
dns_credential = application_host.dns_credential
if application_host.certificate_type == 'dns_challenge' and dns_credential:
host_entry['dns_credential'] = dns_credential.name
+16 -6
View File
@@ -77,11 +77,12 @@ class ApplicationForm(forms.ModelForm):
class ApplicationHostForm(forms.ModelForm):
class Meta:
model = ApplicationHost
fields = ['hostname', 'certificate_type', 'dns_credential']
fields = ['hostname', 'certificate_type', 'dns_credential', 'certificate_fallback']
labels = {
'hostname': _('Hostname'),
'certificate_type': _('Certificate'),
'dns_credential': _('DNS Credential'),
'certificate_fallback': _('Use a self-signed certificate if the trusted certificate cannot be issued'),
}
def clean(self):
@@ -103,9 +104,13 @@ class ApplicationHostForm(forms.ModelForm):
if certificate_type == 'dns_challenge':
if not dns_credential:
self.add_error('dns_credential', _('Select a DNS credential to use the DNS challenge.'))
else:
cleaned_data['dns_credential'] = None
self.add_error('dns_credential', _('Select a DNS credential to use the DNS validation.'))
elif dns_credential:
self.add_error('dns_credential', _('A DNS credential can only be used with the DNS validation.'))
# The fallback keeps WireGuard WebAdmin reachable when the ACME certificate cannot be issued
if self.is_reserved or certificate_type == 'self_signed':
cleaned_data['certificate_fallback'] = True
return cleaned_data
@@ -116,6 +121,7 @@ class ApplicationHostForm(forms.ModelForm):
self.fields['dns_credential'].required = False
if self.is_reserved:
self.fields['hostname'].disabled = True
self.fields['certificate_fallback'].disabled = True
self.helper = FormHelper()
self.helper.layout = Layout(
@@ -124,8 +130,12 @@ class ApplicationHostForm(forms.ModelForm):
css_class='row'
),
Div(
Div('certificate_type', css_class='col-xl-6'),
Div('dns_credential', css_class='col-xl-6'),
Div('certificate_type', css_class='col-md-12'),
Div('dns_credential', css_class='col-md-12'),
css_class='row'
),
Div(
Div('certificate_fallback', css_class='col-md-12'),
css_class='row'
),
Div(
@@ -0,0 +1,23 @@
# Generated by Django 5.2.19 on 2026-11-18 20:17
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('app_gateway', '0010_set_self_signed_for_ip_hosts'),
]
operations = [
migrations.AddField(
model_name='applicationhost',
name='certificate_fallback',
field=models.BooleanField(default=True),
),
migrations.AddField(
model_name='applicationhost',
name='certificate_generation',
field=models.PositiveIntegerField(default=0),
),
]
@@ -0,0 +1,18 @@
# Generated by Django 5.2.19 on 2026-11-18 20:27
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('app_gateway', '0011_host_certificate_fallback_and_generation'),
]
operations = [
migrations.AlterField(
model_name='applicationhost',
name='certificate_type',
field=models.CharField(choices=[('http_challenge', 'Trusted certificate (HTTP validation)'), ('dns_challenge', 'Trusted certificate (DNS validation)'), ('self_signed', 'Self-signed certificate')], default='http_challenge', max_length=16),
),
]
+6 -3
View File
@@ -9,9 +9,9 @@ from gatekeeper.models import GatekeeperGroup, AuthMethod, _unique_slug
RESERVED_APP_NAME = 'wireguard_webadmin'
CERTIFICATE_TYPE_CHOICES = (
('self_signed', _('Self-signed')),
('http_challenge', _('HTTP challenge')),
('dns_challenge', _('DNS challenge')),
('http_challenge', _('Trusted certificate (HTTP validation)')),
('dns_challenge', _('Trusted certificate (DNS validation)')),
('self_signed', _('Self-signed certificate')),
)
@@ -70,6 +70,9 @@ class ApplicationHost(models.Model):
dns_credential = models.ForeignKey(
DnsChallengeCredential, on_delete=models.PROTECT, null=True, blank=True, related_name='hosts'
)
certificate_fallback = models.BooleanField(default=True)
# Incremented to request a new ACME attempt; exported so every Caddy instance applies it
certificate_generation = models.PositiveIntegerField(default=0)
created = models.DateTimeField(auto_now_add=True)
updated = models.DateTimeField(auto_now=True)
+1
View File
@@ -14,6 +14,7 @@ urlpatterns = [
# Application Hosts
path('host/manage/', views.view_manage_application_host, name='manage_application_host'),
path('host/delete/', views.view_delete_application_host, name='delete_application_host'),
path('host/retry_certificate/', views.view_retry_application_host_certificate, name='retry_application_host_certificate'),
# Access Policies
path('policy/select-type/', views.view_select_policy_type, name='select_policy_type'),
+34 -5
View File
@@ -3,7 +3,7 @@ import os
from django.conf import settings
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.db.models import ProtectedError
from django.db.models import F, ProtectedError
from django.shortcuts import render, get_object_or_404, redirect
from django.urls import reverse
from django.utils.translation import gettext as _
@@ -183,11 +183,12 @@ def view_manage_application_host(request):
<h5>Certificate</h5>
<p>Defines how the TLS certificate for this hostname is issued.</p>
<ul>
<li><strong>Self-signed</strong>: A certificate issued by the internal Caddy CA. Browsers will show a security warning.</li>
<li><strong>HTTP challenge</strong>: A trusted ACME certificate validated through HTTP. The hostname must point to this server and ports 80/443 must be reachable from the internet.</li>
<li><strong>DNS challenge</strong>: A trusted ACME certificate validated through a DNS record created using the selected DNS credential. The server does not need to be reachable from the internet.</li>
<li><strong>Trusted certificate (HTTP validation)</strong>: Issued through ACME (e.g. Let's Encrypt) and validated over HTTP. The hostname must point to this server and ports 80/443 must be reachable from the internet.</li>
<li><strong>Trusted certificate (DNS validation)</strong>: Issued through ACME and validated with a DNS record created using the selected DNS credential. The server does not need to be reachable from the internet.</li>
<li><strong>Self-signed certificate</strong>: Issued by the internal Caddy CA. Browsers will show a security warning.</li>
</ul>
<p>If an ACME certificate cannot be issued, a self-signed certificate is used until the next successful attempt.</p>
<p>When the fallback option is enabled and a trusted certificate cannot be issued, a self-signed certificate is used until the next successful attempt. Without the fallback, the connection fails until the certificate is issued.</p>
<p>Use <strong>Retry certificate</strong> in the host list to request a new attempt right away, for example after fixing the DNS or firewall.</p>
''')
if is_reserved:
form_description_content += _('''
@@ -471,6 +472,34 @@ def view_delete_application_route(request):
return render(request, 'generic_delete_confirmation.html', context)
@login_required
@require_POST
def view_retry_application_host_certificate(request):
if not UserAcl.objects.filter(user=request.user).filter(user_level__gte=50).exists():
return render(request, 'access_denied.html', {'page_title': _('Access Denied')})
application_host = get_object_or_404(ApplicationHost, uuid=request.GET.get('uuid'))
redirect_url = reverse('view_application') + f'?uuid={application_host.application.uuid}#hosts'
if application_host.certificate_type == 'self_signed':
messages.error(request, _('This host uses a self-signed certificate.'))
return redirect(redirect_url)
if not settings.CADDY_ENABLED:
messages.error(request, _('This action is not available because Caddy is not enabled.'))
return redirect(redirect_url)
ApplicationHost.objects.filter(pk=application_host.pk).update(certificate_generation=F('certificate_generation') + 1)
export_caddy_config('/caddy_json_export/')
if settings.DEBUG:
export_caddy_config(os.path.join(settings.BASE_DIR, 'containers', 'caddy', 'config_files'))
messages.success(request, _(
'A new certificate will be requested for %(hostname)s. The current configuration was exported.'
) % {'hostname': application_host.hostname})
return redirect(redirect_url)
@login_required
def view_manage_dns_credential(request):
if not UserAcl.objects.filter(user=request.user).filter(user_level__gte=50).exists():
+31 -4
View File
@@ -6,6 +6,7 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PYTHON="python3"
JSON_DIR="${JSON_DIR:-/caddy_json_export}"
CADDYFILE_PATH="${CADDYFILE_PATH:-/etc/caddy/Caddyfile}"
CADDY_PID=""
echo "==> Generating wireguard_webadmin.json..."
"$PYTHON" "${SCRIPT_DIR}/export_wireguard_webadmin_config.py"
@@ -19,12 +20,31 @@ generate_caddyfile() {
fi
}
sync_tls_state() {
"$PYTHON" "${SCRIPT_DIR}/process_config.py" --sync-tls-state
}
start_caddy() {
caddy run --config "$CADDYFILE_PATH" --adapter caddyfile &
CADDY_PID=$!
}
stop_caddy() {
kill "$CADDY_PID" 2>/dev/null || true
for attempt in $(seq 1 10); do
kill -0 "$CADDY_PID" 2>/dev/null || break
sleep 1
done
kill -9 "$CADDY_PID" 2>/dev/null || true
wait "$CADDY_PID" 2>/dev/null || true
}
echo "==> Processing config files..."
generate_caddyfile
sync_tls_state >/dev/null
echo "==> Starting Caddy..."
caddy run --config "$CADDYFILE_PATH" --adapter caddyfile &
CADDY_PID=$!
start_caddy
sleep 2
@@ -36,6 +56,13 @@ while true; do
echo "==> Config change detected, reprocessing..."
generate_caddyfile
echo "==> Reloading Caddy..."
caddy reload --config "$CADDYFILE_PATH" --adapter caddyfile 2>/dev/null || echo "Warning: Caddy reload failed, will retry on next change."
if [ "$(sync_tls_state)" = "restart" ]; then
# A reload keeps previously issued certificates cached in memory
echo "==> Certificate settings changed, restarting Caddy..."
stop_caddy
start_caddy
else
echo "==> Reloading Caddy..."
caddy reload --config "$CADDYFILE_PATH" --adapter caddyfile 2>/dev/null || echo "Warning: Caddy reload failed, will retry on next change."
fi
done
+113 -12
View File
@@ -10,9 +10,12 @@ Expected input files in /caddy_json_export/:
- tls.json (optional, exported from Django)
"""
import hashlib
import json
import os
import re
import shutil
import sys
from urllib.parse import urlparse
JSON_DIR = os.environ.get("JSON_DIR", "/caddy_json_export")
@@ -22,7 +25,15 @@ AUTH_GATEWAY_PORTAL_PATH = os.environ.get("AUTH_GATEWAY_EXTERNAL_PATH", "/auth-g
AUTH_GATEWAY_CHECK_URI = "/auth/check"
CERTIFICATE_TYPES = ("self_signed", "http_challenge", "dns_challenge")
DEFAULT_CERTIFICATE_TYPE = "http_challenge"
ACME_DNS_RESOLVERS = "1.1.1.1 1.0.0.1"
ACME_CERTIFICATE_TYPES = ("http_challenge", "dns_challenge")
# The propagation check queries DNS from this server, which fails on networks that intercept
# port 53 traffic. Waiting a fixed delay and letting the CA validate from outside is more reliable.
ACME_DNS_PROPAGATION_DELAY = "30s"
CADDY_DATA_DIR = os.path.join(os.environ.get("XDG_DATA_HOME", "/data"), "caddy")
CERTIFICATES_DIR = os.path.join(CADDY_DATA_DIR, "certificates")
INTERNAL_ISSUER_DIR_NAME = "local"
TLS_STATE_PATH = os.path.join(CADDY_DATA_DIR, "wireguard_webadmin_tls_state.json")
TLS_STATE_PENDING_PATH = f"{CADDYFILE_PATH}.tls_state.json"
# Set by entrypoint.sh when the generated Caddyfile fails validation
DISABLE_DNS_CHALLENGE = os.environ.get("DISABLE_DNS_CHALLENGE", "") == "1"
AUTH_IDENTITY_HEADERS = (
@@ -82,7 +93,10 @@ def _safe_identifier(val: str) -> str:
def get_host_tls(host, tls_data):
"""Return (certificate_type, dns_credential_name) for a host. Hosts not listed use the HTTP challenge."""
"""
Return (certificate_type, dns_credential_name, certificate_fallback) for a host.
Hosts not listed use the HTTP challenge with fallback.
"""
host_tls = tls_data.get("hosts", {}).get(host.lower(), {})
certificate_type = host_tls.get("certificate_type", DEFAULT_CERTIFICATE_TYPE)
if certificate_type not in CERTIFICATE_TYPES:
@@ -94,7 +108,9 @@ def get_host_tls(host, tls_data):
if DISABLE_DNS_CHALLENGE or dns_credential not in tls_data.get("dns_credentials", {}):
certificate_type = DEFAULT_CERTIFICATE_TYPE
dns_credential = None
return certificate_type, dns_credential
certificate_fallback = certificate_type == "self_signed" or host_tls.get("certificate_fallback", True) is not False
return certificate_type, dns_credential, certificate_fallback
def group_hosts_by_tls(hosts, tls_data):
@@ -122,7 +138,7 @@ def build_caddyfile(apps, auth_policies, routes, tls_data=None):
lines.append(" }")
lines.append("")
def emit_tls(certificate_type, dns_credential):
def emit_tls(certificate_type, dns_credential, certificate_fallback):
if certificate_type == "self_signed":
lines.append(" tls internal")
lines.append("")
@@ -137,12 +153,14 @@ def build_caddyfile(apps, auth_policies, routes, tls_data=None):
for field_name, field_value in dns_credential_data.get("credentials", {}).items():
lines.append(f" {_safe_identifier(field_name)} {_quote_value(field_value)}")
lines.append(" }")
lines.append(f" resolvers {ACME_DNS_RESOLVERS}")
lines.append(f" propagation_delay {ACME_DNS_PROPAGATION_DELAY}")
lines.append(" propagation_timeout -1")
lines.append(" }")
else:
lines.append(" issuer acme")
# Fall back to the internal CA when the ACME certificate cannot be issued
lines.append(" issuer internal")
if certificate_fallback:
# Fall back to the internal CA when the ACME certificate cannot be issued
lines.append(" issuer internal")
lines.append(" }")
lines.append("")
@@ -206,7 +224,7 @@ def build_caddyfile(apps, auth_policies, routes, tls_data=None):
lines.append(" }")
lines.append("")
def emit_site_block(app, hosts, certificate_type, dns_credential):
def emit_site_block(app, hosts, certificate_type, dns_credential, certificate_fallback):
upstream = app.get("upstream", "")
# tls_insecure_skip_verify is only valid for https upstreams
allow_invalid_cert = app.get("allow_invalid_cert", False) and upstream.startswith("https://")
@@ -217,7 +235,7 @@ def build_caddyfile(apps, auth_policies, routes, tls_data=None):
safe_hosts = [_safe_value(h) for h in hosts]
lines.append(f"{', '.join(safe_hosts)} {{")
emit_tls(certificate_type, dns_credential)
emit_tls(certificate_type, dns_credential, certificate_fallback)
lines.append(" # Security: overwrite client-supplied forwarding headers with verified values")
lines.append(" request_header X-Forwarded-For {http.request.remote.host}")
lines.append(" request_header -X-Forwarded-Host")
@@ -278,12 +296,91 @@ def build_caddyfile(apps, auth_policies, routes, tls_data=None):
if not hosts or not app.get("upstream", ""):
continue
for (certificate_type, dns_credential), group_hosts in group_hosts_by_tls(hosts, tls_data):
emit_site_block(app, group_hosts, certificate_type, dns_credential)
for (certificate_type, dns_credential, certificate_fallback), group_hosts in group_hosts_by_tls(hosts, tls_data):
emit_site_block(app, group_hosts, certificate_type, dns_credential, certificate_fallback)
return "\n".join(lines)
def build_tls_state(apps, tls_data):
"""Map each host to a fingerprint of the certificate settings used in the generated Caddyfile."""
tls_data = tls_data or {}
tls_state = {}
for app in apps:
if not app.get("hosts") or not app.get("upstream"):
continue
for host in app["hosts"]:
certificate_type, dns_credential, certificate_fallback = get_host_tls(host, tls_data)
credential_hash = ""
if dns_credential:
credential_data = json.dumps(tls_data["dns_credentials"][dns_credential], sort_keys=True)
credential_hash = hashlib.sha256(credential_data.encode()).hexdigest()
certificate_generation = tls_data.get("hosts", {}).get(host.lower(), {}).get("certificate_generation", 0)
tls_state[host.lower()] = [
certificate_type, dns_credential or "", credential_hash, certificate_fallback, certificate_generation
]
return tls_state
def _write_json(filepath, data):
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, "w", encoding="utf-8") as json_file:
json.dump(data, json_file)
def _read_json(filepath):
if not os.path.exists(filepath):
return None
with open(filepath, "r", encoding="utf-8") as json_file:
return json.load(json_file)
def has_acme_certificate(host):
if not os.path.isdir(CERTIFICATES_DIR):
return False
for issuer_dir_name in os.listdir(CERTIFICATES_DIR):
if issuer_dir_name == INTERNAL_ISSUER_DIR_NAME:
continue
if os.path.isfile(os.path.join(CERTIFICATES_DIR, issuer_dir_name, host, f"{host}.crt")):
return True
return False
def sync_tls_state():
"""
Compare the certificate settings of the last generated Caddyfile with the ones currently applied.
For hosts whose ACME settings changed (including a retry request), the fallback certificate from the
internal CA is removed so a new ACME certificate is requested right away instead of on the next renewal.
Prints "restart" when Caddy must be restarted, since a reload keeps cached certificates in memory. Hosts
that already hold an ACME certificate and had no fallback certificate removed do not require a restart.
"""
pending_state = _read_json(TLS_STATE_PENDING_PATH)
if pending_state is None:
return
applied_state = _read_json(TLS_STATE_PATH)
restart_required = False
if applied_state is not None:
for host, host_state in pending_state.items():
if host_state[0] not in ACME_CERTIFICATE_TYPES or applied_state.get(host) in (None, host_state):
continue
if "/" in host or host in ("", ".", ".."):
continue
fallback_removed = False
fallback_certificate_dir = os.path.join(CERTIFICATES_DIR, INTERNAL_ISSUER_DIR_NAME, host)
if os.path.isdir(fallback_certificate_dir):
shutil.rmtree(fallback_certificate_dir)
fallback_removed = True
print(f"Removed fallback certificate for {host}", file=sys.stderr)
if fallback_removed or not has_acme_certificate(host):
restart_required = True
_write_json(TLS_STATE_PATH, pending_state)
if restart_required:
print("restart")
def main():
apps = collect_all_applications()
auth_policies = load_json("auth_policies.json")
@@ -294,8 +391,12 @@ def main():
os.makedirs(os.path.dirname(CADDYFILE_PATH), exist_ok=True)
with open(CADDYFILE_PATH, "w", encoding="utf-8") as caddyfile:
caddyfile.write(caddyfile_content)
_write_json(TLS_STATE_PENDING_PATH, build_tls_state(apps, tls_data))
print(f"Caddyfile written to {CADDYFILE_PATH}")
if __name__ == "__main__":
main()
if "--sync-tls-state" in sys.argv:
sync_tls_state()
else:
main()
+12 -1
View File
@@ -84,8 +84,19 @@
<td>
{{ host.get_certificate_type_display }}
{% if host.dns_credential %}<small class="text-muted">({{ host.dns_credential }})</small>{% endif %}
{% if host.certificate_type != 'self_signed' and not host.certificate_fallback %}
<small class="text-muted">({% trans 'no fallback' %})</small>
{% endif %}
</td>
<td style="width: 15%">
<td style="width: 15%" class="text-nowrap">
{% if host.certificate_type != 'self_signed' %}
<form method="post" action="{% url 'retry_application_host_certificate' %}?uuid={{ host.uuid }}" class="d-inline">
{% csrf_token %}
<button type="submit" class="btn btn-sm btn-outline-secondary" title="{% trans 'Retry certificate' %}">
<i class="fas fa-sync-alt"></i>
</button>
</form>
{% endif %}
{% if is_reserved %}
<a href="{% url 'manage_application_host' %}?uuid={{ host.uuid }}"
class="btn btn-sm btn-info" title="{% trans 'Edit' %}">
@@ -25,7 +25,7 @@
<p>{% trans 'Credentials used to create the DNS records required by the ACME DNS challenge.' %}</p>
<p>{% blocktrans trimmed %}
Use a token restricted to editing DNS records of the zones you need. For Cloudflare, create an
API Token with the <strong>Zone / DNS / Edit</strong> permission.
API Token with the <strong>Zone / Zone / Read</strong> and <strong>Zone / DNS / Edit</strong> permissions.
{% endblocktrans %}</p>
<p>{% trans 'Secret values are never displayed after saving. Leave them blank when editing to keep the current value.' %}</p>
</div>