From c436d291823ade20da9fe3d3cb61d66c4d79ee5e Mon Sep 17 00:00:00 2001 From: cyberops7 <18562612+cyberops7@users.noreply.github.com> Date: Wed, 15 Jul 2026 22:26:45 -0600 Subject: [PATCH] K3S-Deploy: harden HA bring-up and expand README caveats Add a post-join control-plane readiness gate so a half-succeeded server join on a degraded etcd can no longer pass as success. Retry the MetalLB IPAddressPool/L2Advertisement applies to ride out the validating-webhook startup race. Narrow the k3sup cleanup glob, quiet the final k3sup ready tip, and clarify two comments. README: add networking caveats (VIP/LB outside the DHCP pool, same-subnet ARP/L2, interface naming), certName key placement, a post-run verification section, machine-count/admin-box notes, and fix the RAW_BASE description. --- Kubernetes/K3S-Deploy/README.md | 58 +++++++++++++++++++++++++++++---- Kubernetes/K3S-Deploy/k3s.sh | 44 +++++++++++++++++++++---- 2 files changed, 89 insertions(+), 13 deletions(-) diff --git a/Kubernetes/K3S-Deploy/README.md b/Kubernetes/K3S-Deploy/README.md index a420f19..7d6df7b 100644 --- a/Kubernetes/K3S-Deploy/README.md +++ b/Kubernetes/K3S-Deploy/README.md @@ -38,26 +38,71 @@ duplicate LoadBalancer controllers. - An **SSH key pair** you can use to reach the nodes (the script distributes the public key and never clobbers your `~/.ssh/config`). -- At least **3 control-plane nodes** for real HA (etcd needs a quorum). -- Run the script from an **Ubuntu/Debian admin box** (it installs `k3sup` - and `kubectl` locally if missing). +- **5 nodes by default** (3 servers + 2 agents). The minimum for HA is 3 + servers (etcd needs a quorum); workers are optional. Adjust the node list in + the config block if you have fewer machines. +- Run the script from a **separate Ubuntu/Debian admin machine** (your + laptop/workstation) that can SSH to every node — not on one of the nodes + themselves. It installs `k3sup` and `kubectl` locally if missing. + +## Important caveats + +- **The VIP and the MetalLB range must be outside your DHCP pool**, and must + not overlap each other or any node IP. If your router hands out `vip` or an + address in `lbrange` via DHCP, you'll get intermittent, hard-to-debug + failures — reserve them on your router/DHCP server first. +- **Same subnet / L2 only.** kube-vip (ARP mode) and MetalLB (L2) both + advertise via ARP, so the VIP and LoadBalancer IPs must be on the **same + subnet/VLAN** as the nodes and the clients reaching them — they are not + routed across subnets. +- **`interface` must match your nodes' real NIC.** It's `eth0` on many + systems but often `ens18`/`enp0s3` on Proxmox and cloud images. Check with + `ip -o -4 route show default`. +- **Host keys are trusted on first use** (`accept-new`). If you re-image a + node, clear its stale entry first: `ssh-keygen -R `. +- This is a **homelab tutorial, not production-hardened** as-is (aggressive + leader election, no etcd backups, single L2 domain). ## Usage -1. Snapshot your VMs. -2. Copy your SSH key into your home directory (or into `~/.ssh`). +1. Snapshot your VMs (so you can roll back — the script modifies every node). +2. Place your SSH **private** key at `~/` or `~/.ssh/`, + with the matching `.pub` beside it. `certName` is the key's *filename only* + — no path, no `.pub` suffix. Modern OpenSSH defaults to `id_ed25519`, so set + `certName` to match the key you actually have. 3. Edit the **"YOU SHOULD ONLY NEED TO EDIT THIS SECTION"** block in `k3s.sh` — node IPs, `user`, `interface`, `vip`, `lbrange`, `certName`. + The example IPs (`192.168.3.x`, VIP `.50`, range `.60-.80`) and + `interface=eth0` are placeholders — change them for your network. 4. `chmod +x k3s.sh && ./k3s.sh` 5. Review the pre-flight summary and confirm. Grab a coffee. +### After it finishes + +The kubeconfig is **merged** into `~/.kube/config` as context `k3s-ha` (set by +`context` in the config block), pointed at the VIP. Select it and check the +cluster: + +```bash +kubectl config use-context k3s-ha +kubectl get nodes -o wide +curl http:// # the script prints the assigned IP +``` + +Worker nodes are labeled `worker=true` and `longhorn=true` (the latter for the +companion Longhorn storage tutorial — harmless if you don't use it). + +If a node fails to join, reset just that node and re-run the script (it's +idempotent): `k3s-uninstall.sh` on a server, `k3s-agent-uninstall.sh` on a +worker. + ### Options (environment variables) | Variable | Purpose | | ---------------- | ----------------------------------------------------- | | `ASSUME_YES=1` | Skip the pre-flight `[y/N]` prompt (unattended runs). | | `NO_COLOR=1` | Disable colored output. | -| `RAW_BASE=` | Base URL for the sibling manifests (default: `main`). | +| `RAW_BASE=` | Where to fetch the sibling manifests (kube-vip, ipAddressPool, l2Advertisement); default: upstream `main`. Override for a fork/branch/local copy. | ### Tracking the latest k3s instead of a pinned version @@ -75,7 +120,6 @@ docker run --rm ghcr.io/kube-vip/kube-vip: manifest daemonset \ --controlplane --arp --leaderElection --taint --inCluster > kube-vip sed -i 's/value: eth0/value: REPLACE_INTERFACE/' kube-vip sed -i 's/value: 10.0.0.254/value: REPLACE_VIP/' kube-vip - ``` `--inCluster` is required on k3s: it makes kube-vip use the `kube-vip` diff --git a/Kubernetes/K3S-Deploy/k3s.sh b/Kubernetes/K3S-Deploy/k3s.sh index 8383958..5e39324 100755 --- a/Kubernetes/K3S-Deploy/k3s.sh +++ b/Kubernetes/K3S-Deploy/k3s.sh @@ -120,6 +120,21 @@ server_extra_args() { # $1 = node ip # never node-token (whose stdout is captured for the token). k3sup_quiet() { command k3sup "$@" 2>&1 | sed '/Create clusters on Mac/d'; } +# Apply a manifest, retrying on transient failure. MetalLB installs a +# validating webhook served by its controller; there is a brief window where +# the controller Deployment reports Available but the webhook endpoints/TLS +# are not serving yet, so the first apply can fail with "no endpoints +# available for service metallb-webhook-service". Retry to ride that out. +kubectl_apply_retry() { # $1 = manifest path or URL + local i + for i in $(seq 1 6); do + kubectl apply -f "$1" && return 0 + warn "kubectl apply $1 failed (attempt $i/6) — retrying in 5s" + sleep 5 + done + die "kubectl apply $1 failed after 6 attempts" +} + # ── Pre-flight: show config, confirm before touching any node ────────── step "Pre-flight · review configuration" printf ' %-16s %s\n' \ @@ -153,7 +168,7 @@ if ! command -v k3sup &>/dev/null; then done [[ -n "$k3sup_bin" ]] || die "k3sup installer produced no binary" sudo install "$k3sup_bin" /usr/local/bin/k3sup - rm -f k3sup k3sup-* + rm -f "$k3sup_bin" else ok "k3sup present" fi @@ -269,8 +284,10 @@ REMOTE done ok "kube-vip deployed to ${#masters_all[@]} control-plane nodes" info "Pointing local kubeconfig at the VIP ($vip)" -# Rewrite only the server URL (anchored) to avoid the regex dots in the IP -# matching anything else in the kubeconfig (e.g. base64 cert data). +# Rewrite the server URL from master1 to the VIP. The dots in the IP are +# unescaped regex metacharacters, but a false match would need the literal +# "https://:6443" string elsewhere in the kubeconfig, which does +# not occur (cert data is base64 — no scheme or port), so this is safe here. sed -i "s#https://$master1:6443#https://$vip:6443#" "$HOME/.kube/config" # Confirm the VIP is actually answering before any VIP-routed kubectl below. @@ -296,6 +313,9 @@ ok "Join token fetched" # ── Step 7/9 · Join control-plane and worker nodes ───────────────────── step "Step 7/9 · Join control-plane and worker nodes" +# The prefetched --node-token is what keeps these joins from ever SSHing to +# the server: without it, k3sup would try --server-user (default: root) against +# --server-ip (the VIP, not an SSH target) and fail confusingly. Keep the token. for node in "${masters[@]}"; do info "Joining control-plane node $node" k3sup_quiet join \ @@ -324,6 +344,18 @@ for node in "${workers[@]}"; do ok "Worker node $node joined" done +# Assert the full HA control plane is up before continuing. The VIP gate in +# Step 5 only proved master1 (a single-member etcd); a control-plane join that +# half-succeeds would otherwise sail through as success on a degraded cluster. +info "Waiting for all ${#masters_all[@]} control-plane nodes to be Ready" +kubectl wait --for=condition=Ready node \ + -l node-role.kubernetes.io/control-plane --timeout=180s || \ + die "Control-plane nodes did not all become Ready — check etcd/kube-vip on the masters: kubectl get nodes; kubectl -n kube-system logs -l app.kubernetes.io/name=kube-vip-ds" +cp_ready="$(kubectl get nodes -l node-role.kubernetes.io/control-plane --no-headers 2>/dev/null | awk '$2=="Ready"{c++} END{print c+0}')" +[[ "$cp_ready" -eq "${#masters_all[@]}" ]] || \ + die "Expected ${#masters_all[@]} control-plane nodes Ready, found $cp_ready — etcd quorum may be degraded" +ok "All ${#masters_all[@]} control-plane nodes Ready" + # ── Step 8/9 · Install MetalLB (service LoadBalancer) ────────────────── step "Step 8/9 · Install MetalLB (service LoadBalancer)" kubectl apply -f "https://raw.githubusercontent.com/metallb/metallb/$METALLB_VERSION/config/manifests/metallb-native.yaml" @@ -335,15 +367,15 @@ kubectl -n metallb-system rollout status deploy/controller --timeout=120s info "Configuring address pool ($lbrange)" curl -sfL "$manifest_base/ipAddressPool" -o "$HOME/ipAddressPool.src" sed "s|REPLACE_LBRANGE|$lbrange|g" "$HOME/ipAddressPool.src" > "$HOME/ipAddressPool.yaml" -kubectl apply -f "$HOME/ipAddressPool.yaml" +kubectl_apply_retry "$HOME/ipAddressPool.yaml" curl -sfL "$manifest_base/l2Advertisement.yaml" -o "$HOME/l2Advertisement.yaml" -kubectl apply -f "$HOME/l2Advertisement.yaml" +kubectl_apply_retry "$HOME/l2Advertisement.yaml" ok "MetalLB configured" # ── Step 9/9 · Verify cluster and LoadBalancer ───────────────────────── step "Step 9/9 · Verify cluster and LoadBalancer" info "Waiting for the cluster to be ready" -k3sup ready --context "$context" --kubeconfig "$HOME/.kube/config" +k3sup_quiet ready --context "$context" --kubeconfig "$HOME/.kube/config" info "Deploying nginx sample" kubectl apply -n default -f https://raw.githubusercontent.com/inlets/inlets-operator/master/contrib/nginx-sample-deployment.yaml # `|| true` — expose is imperative and errors with AlreadyExists on a re-run,