Update notification service

AppImage/components/notification-settings.tsx

@@ -0,0 +1,827 @@
+"use client"
+
+import { useState, useEffect, useCallback } from "react"
+import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "./ui/card"
+import { Tabs, TabsList, TabsTrigger, TabsContent } from "./ui/tabs"
+import { Input } from "./ui/input"
+import { Label } from "./ui/label"
+import { Badge } from "./ui/badge"
+import { Checkbox } from "./ui/checkbox"
+import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "./ui/select"
+import { fetchApi } from "../lib/api-config"
+import {
+  Bell, BellOff, Send, CheckCircle2, XCircle, Loader2,
+  AlertTriangle, Info, Settings2, Zap, Eye, EyeOff,
+  Trash2, ChevronDown, ChevronUp, TestTube2
+} from "lucide-react"
+
+interface ChannelConfig {
+  enabled: boolean
+  bot_token?: string
+  chat_id?: string
+  url?: string
+  token?: string
+  webhook_url?: string
+}
+
+interface NotificationConfig {
+  enabled: boolean
+  channels: Record<string, ChannelConfig>
+  severity_filter: string
+  event_categories: Record<string, boolean>
+  ai_enabled: boolean
+  ai_provider: string
+  ai_api_key: string
+  ai_model: string
+  hostname: string
+}
+
+interface ServiceStatus {
+  enabled: boolean
+  running: boolean
+  channels: Record<string, boolean>
+  queue_size: number
+  last_sent: string | null
+  total_sent_24h: number
+}
+
+interface HistoryEntry {
+  id: number
+  event_type: string
+  channel: string
+  title: string
+  severity: string
+  sent_at: string
+  success: boolean
+  error_message: string | null
+}
+
+const SEVERITY_OPTIONS = [
+  { value: "critical", label: "Critical only" },
+  { value: "warning", label: "Warning + Critical" },
+  { value: "info", label: "All (Info + Warning + Critical)" },
+]
+
+const EVENT_CATEGORIES = [
+  { key: "system", label: "System", desc: "Startup, shutdown, kernel events" },
+  { key: "vm_ct", label: "VM / CT", desc: "Start, stop, crash, migration" },
+  { key: "backup", label: "Backups", desc: "Backup start, complete, fail" },
+  { key: "resources", label: "Resources", desc: "CPU, memory, temperature" },
+  { key: "storage", label: "Storage", desc: "Disk space, I/O errors, SMART" },
+  { key: "network", label: "Network", desc: "Connectivity, bond, latency" },
+  { key: "security", label: "Security", desc: "Auth failures, fail2ban, firewall" },
+  { key: "cluster", label: "Cluster", desc: "Quorum, split-brain, HA fencing" },
+]
+
+const AI_PROVIDERS = [
+  { value: "openai", label: "OpenAI" },
+  { value: "groq", label: "Groq" },
+]
+
+const DEFAULT_CONFIG: NotificationConfig = {
+  enabled: false,
+  channels: {
+    telegram: { enabled: false },
+    gotify: { enabled: false },
+    discord: { enabled: false },
+  },
+  severity_filter: "warning",
+  event_categories: {
+    system: true, vm_ct: true, backup: true, resources: true,
+    storage: true, network: true, security: true, cluster: true,
+  },
+  ai_enabled: false,
+  ai_provider: "openai",
+  ai_api_key: "",
+  ai_model: "",
+  hostname: "",
+}
+
+export function NotificationSettings() {
+  const [config, setConfig] = useState<NotificationConfig>(DEFAULT_CONFIG)
+  const [status, setStatus] = useState<ServiceStatus | null>(null)
+  const [history, setHistory] = useState<HistoryEntry[]>([])
+  const [loading, setLoading] = useState(true)
+  const [saving, setSaving] = useState(false)
+  const [saved, setSaved] = useState(false)
+  const [testing, setTesting] = useState<string | null>(null)
+  const [testResult, setTestResult] = useState<{ channel: string; success: boolean; message: string } | null>(null)
+  const [showHistory, setShowHistory] = useState(false)
+  const [showAdvanced, setShowAdvanced] = useState(false)
+  const [showSecrets, setShowSecrets] = useState<Record<string, boolean>>({})
+  const [editMode, setEditMode] = useState(false)
+  const [hasChanges, setHasChanges] = useState(false)
+  const [originalConfig, setOriginalConfig] = useState<NotificationConfig>(DEFAULT_CONFIG)
+
+  const loadConfig = useCallback(async () => {
+    try {
+      const data = await fetchApi<{ success: boolean; config: NotificationConfig }>("/api/notifications/settings")
+      if (data.success && data.config) {
+        setConfig(data.config)
+        setOriginalConfig(data.config)
+      }
+    } catch (err) {
+      console.error("Failed to load notification settings:", err)
+    } finally {
+      setLoading(false)
+    }
+  }, [])
+
+  const loadStatus = useCallback(async () => {
+    try {
+      const data = await fetchApi<{ success: boolean } & ServiceStatus>("/api/notifications/status")
+      if (data.success) {
+        setStatus(data)
+      }
+    } catch {
+      // Service may not be running yet
+    }
+  }, [])
+
+  const loadHistory = useCallback(async () => {
+    try {
+      const data = await fetchApi<{ success: boolean; history: HistoryEntry[]; total: number }>("/api/notifications/history?limit=20")
+      if (data.success) {
+        setHistory(data.history || [])
+      }
+    } catch {
+      // Ignore
+    }
+  }, [])
+
+  useEffect(() => {
+    loadConfig()
+    loadStatus()
+  }, [loadConfig, loadStatus])
+
+  useEffect(() => {
+    if (showHistory) loadHistory()
+  }, [showHistory, loadHistory])
+
+  const updateConfig = (updater: (prev: NotificationConfig) => NotificationConfig) => {
+    setConfig(prev => {
+      const next = updater(prev)
+      setHasChanges(true)
+      return next
+    })
+  }
+
+  const updateChannel = (channel: string, field: string, value: string | boolean) => {
+    updateConfig(prev => ({
+      ...prev,
+      channels: {
+        ...prev.channels,
+        [channel]: { ...prev.channels[channel], [field]: value },
+      },
+    }))
+  }
+
+  const handleSave = async () => {
+    setSaving(true)
+    try {
+      await fetchApi("/api/notifications/settings", {
+        method: "POST",
+        body: JSON.stringify(config),
+      })
+      setOriginalConfig(config)
+      setHasChanges(false)
+      setEditMode(false)
+      setSaved(true)
+      setTimeout(() => setSaved(false), 3000)
+      loadStatus()
+    } catch (err) {
+      console.error("Failed to save notification settings:", err)
+    } finally {
+      setSaving(false)
+    }
+  }
+
+  const handleCancel = () => {
+    setConfig(originalConfig)
+    setHasChanges(false)
+    setEditMode(false)
+  }
+
+  const handleTest = async (channel: string) => {
+    setTesting(channel)
+    setTestResult(null)
+    try {
+      const data = await fetchApi<{ success: boolean; message: string }>("/api/notifications/test", {
+        method: "POST",
+        body: JSON.stringify({ channel }),
+      })
+      setTestResult({ channel, success: data.success, message: data.message })
+    } catch (err) {
+      setTestResult({ channel, success: false, message: String(err) })
+    } finally {
+      setTesting(null)
+      setTimeout(() => setTestResult(null), 5000)
+    }
+  }
+
+  const handleClearHistory = async () => {
+    try {
+      await fetchApi("/api/notifications/history", { method: "DELETE" })
+      setHistory([])
+    } catch {
+      // Ignore
+    }
+  }
+
+  const toggleSecret = (key: string) => {
+    setShowSecrets(prev => ({ ...prev, [key]: !prev[key] }))
+  }
+
+  if (loading) {
+    return (
+      <Card>
+        <CardHeader>
+          <div className="flex items-center gap-2">
+            <Bell className="h-5 w-5 text-blue-500" />
+            <CardTitle>Notifications</CardTitle>
+          </div>
+        </CardHeader>
+        <CardContent>
+          <div className="flex items-center justify-center py-8">
+            <div className="animate-spin h-8 w-8 border-4 border-blue-500 border-t-transparent rounded-full" />
+          </div>
+        </CardContent>
+      </Card>
+    )
+  }
+
+  const activeChannels = Object.entries(config.channels).filter(([, ch]) => ch.enabled).length
+
+  return (
+    <Card>
+      <CardHeader>
+        <div className="flex items-center justify-between">
+          <div className="flex items-center gap-2">
+            <Bell className="h-5 w-5 text-blue-500" />
+            <CardTitle>Notifications</CardTitle>
+            {config.enabled && (
+              <Badge variant="outline" className="text-[10px] border-green-500/30 text-green-500">
+                Active
+              </Badge>
+            )}
+          </div>
+          <div className="flex items-center gap-2">
+            {saved && (
+              <span className="flex items-center gap-1 text-xs text-green-500">
+                <CheckCircle2 className="h-3.5 w-3.5" />
+                Saved
+              </span>
+            )}
+            {editMode ? (
+              <>
+                <button
+                  className="h-7 px-3 text-xs rounded-md border border-border bg-background hover:bg-muted transition-colors text-muted-foreground"
+                  onClick={handleCancel}
+                  disabled={saving}
+                >
+                  Cancel
+                </button>
+                <button
+                  className="h-7 px-3 text-xs rounded-md bg-blue-600 hover:bg-blue-700 text-white transition-colors disabled:opacity-50 flex items-center gap-1.5"
+                  onClick={handleSave}
+                  disabled={saving || !hasChanges}
+                >
+                  {saving ? <Loader2 className="h-3 w-3 animate-spin" /> : <CheckCircle2 className="h-3 w-3" />}
+                  Save
+                </button>
+              </>
+            ) : (
+              <button
+                className="h-7 px-3 text-xs rounded-md border border-border bg-background hover:bg-muted transition-colors flex items-center gap-1.5"
+                onClick={() => setEditMode(true)}
+              >
+                <Settings2 className="h-3 w-3" />
+                Edit
+              </button>
+            )}
+          </div>
+        </div>
+        <CardDescription>
+          Configure notification channels and event filters. Receive alerts via Telegram, Gotify, or Discord.
+        </CardDescription>
+      </CardHeader>
+
+      <CardContent className="space-y-5">
+        {/* ── Service Status ── */}
+        {status && (
+          <div className="flex items-center gap-3 p-3 rounded-lg bg-muted/50 border border-border">
+            <div className={`h-2.5 w-2.5 rounded-full ${status.running ? "bg-green-500" : "bg-red-500"}`} />
+            <div className="flex-1 min-w-0">
+              <span className="text-xs font-medium">
+                {status.running ? "Service running" : "Service stopped"}
+              </span>
+              {status.total_sent_24h > 0 && (
+                <span className="text-xs text-muted-foreground ml-2">
+                  {status.total_sent_24h} sent in last 24h
+                </span>
+              )}
+            </div>
+            {activeChannels > 0 && (
+              <Badge variant="outline" className="text-[10px]">
+                {activeChannels} channel{activeChannels > 1 ? "s" : ""}
+              </Badge>
+            )}
+          </div>
+        )}
+
+        {/* ── Enable/Disable ── */}
+        <div className="flex items-center justify-between py-2 px-1">
+          <div className="flex items-center gap-2">
+            {config.enabled ? (
+              <Bell className="h-4 w-4 text-blue-500" />
+            ) : (
+              <BellOff className="h-4 w-4 text-muted-foreground" />
+            )}
+            <div>
+              <span className="text-sm font-medium">Enable Notifications</span>
+              <p className="text-[11px] text-muted-foreground">Activate the notification service</p>
+            </div>
+          </div>
+          <button
+            className={`relative w-10 h-5 rounded-full transition-colors ${
+              config.enabled ? "bg-blue-600" : "bg-muted-foreground/30"
+            } ${!editMode ? "opacity-60 cursor-not-allowed" : "cursor-pointer"}`}
+            onClick={() => editMode && updateConfig(p => ({ ...p, enabled: !p.enabled }))}
+            disabled={!editMode}
+            role="switch"
+            aria-checked={config.enabled}
+          >
+            <span
+              className={`absolute top-0.5 left-0.5 h-4 w-4 rounded-full bg-white shadow transition-transform ${
+                config.enabled ? "translate-x-5" : "translate-x-0"
+              }`}
+            />
+          </button>
+        </div>
+
+        {config.enabled && (
+          <>
+            {/* ── Channel Configuration ── */}
+            <div className="space-y-3">
+              <div className="flex items-center gap-2">
+                <Zap className="h-3.5 w-3.5 text-muted-foreground" />
+                <span className="text-xs font-medium text-muted-foreground uppercase tracking-wider">Channels</span>
+              </div>
+
+              <Tabs defaultValue="telegram" className="w-full">
+                <TabsList className="w-full grid grid-cols-3 h-8">
+                  <TabsTrigger value="telegram" className="text-xs data-[state=active]:text-blue-500">
+                    Telegram
+                  </TabsTrigger>
+                  <TabsTrigger value="gotify" className="text-xs data-[state=active]:text-green-500">
+                    Gotify
+                  </TabsTrigger>
+                  <TabsTrigger value="discord" className="text-xs data-[state=active]:text-indigo-500">
+                    Discord
+                  </TabsTrigger>
+                </TabsList>
+
+                {/* Telegram */}
+                <TabsContent value="telegram" className="space-y-3 pt-2">
+                  <div className="flex items-center justify-between">
+                    <Label className="text-xs font-medium">Enable Telegram</Label>
+                    <button
+                      className={`relative w-9 h-[18px] rounded-full transition-colors ${
+                        config.channels.telegram?.enabled ? "bg-blue-600" : "bg-muted-foreground/30"
+                      } ${!editMode ? "opacity-60 cursor-not-allowed" : "cursor-pointer"}`}
+                      onClick={() => editMode && updateChannel("telegram", "enabled", !config.channels.telegram?.enabled)}
+                      disabled={!editMode}
+                      role="switch"
+                      aria-checked={config.channels.telegram?.enabled || false}
+                    >
+                      <span className={`absolute top-[1px] left-[1px] h-4 w-4 rounded-full bg-white shadow transition-transform ${
+                        config.channels.telegram?.enabled ? "translate-x-[18px]" : "translate-x-0"
+                      }`} />
+                    </button>
+                  </div>
+                  {config.channels.telegram?.enabled && (
+                    <>
+                      <div className="space-y-1.5">
+                        <Label className="text-[11px] text-muted-foreground">Bot Token</Label>
+                        <div className="flex items-center gap-1.5">
+                          <Input
+                            type={showSecrets["tg_token"] ? "text" : "password"}
+                            className="h-7 text-xs font-mono"
+                            placeholder="123456:ABC-DEF1234..."
+                            value={config.channels.telegram?.bot_token || ""}
+                            onChange={e => updateChannel("telegram", "bot_token", e.target.value)}
+                            disabled={!editMode}
+                          />
+                          <button
+                            className="h-7 w-7 flex items-center justify-center rounded-md border border-border hover:bg-muted transition-colors shrink-0"
+                            onClick={() => toggleSecret("tg_token")}
+                          >
+                            {showSecrets["tg_token"] ? <EyeOff className="h-3 w-3" /> : <Eye className="h-3 w-3" />}
+                          </button>
+                        </div>
+                      </div>
+                      <div className="space-y-1.5">
+                        <Label className="text-[11px] text-muted-foreground">Chat ID</Label>
+                        <Input
+                          className="h-7 text-xs font-mono"
+                          placeholder="-1001234567890"
+                          value={config.channels.telegram?.chat_id || ""}
+                          onChange={e => updateChannel("telegram", "chat_id", e.target.value)}
+                          disabled={!editMode}
+                        />
+                      </div>
+                      {!editMode && config.channels.telegram?.bot_token && (
+                        <button
+                          className="h-7 px-3 text-xs rounded-md border border-border bg-background hover:bg-muted transition-colors flex items-center gap-1.5 w-full justify-center"
+                          onClick={() => handleTest("telegram")}
+                          disabled={testing === "telegram"}
+                        >
+                          {testing === "telegram" ? (
+                            <Loader2 className="h-3 w-3 animate-spin" />
+                          ) : (
+                            <TestTube2 className="h-3 w-3" />
+                          )}
+                          Test Telegram
+                        </button>
+                      )}
+                    </>
+                  )}
+                </TabsContent>
+
+                {/* Gotify */}
+                <TabsContent value="gotify" className="space-y-3 pt-2">
+                  <div className="flex items-center justify-between">
+                    <Label className="text-xs font-medium">Enable Gotify</Label>
+                    <button
+                      className={`relative w-9 h-[18px] rounded-full transition-colors ${
+                        config.channels.gotify?.enabled ? "bg-green-600" : "bg-muted-foreground/30"
+                      } ${!editMode ? "opacity-60 cursor-not-allowed" : "cursor-pointer"}`}
+                      onClick={() => editMode && updateChannel("gotify", "enabled", !config.channels.gotify?.enabled)}
+                      disabled={!editMode}
+                      role="switch"
+                      aria-checked={config.channels.gotify?.enabled || false}
+                    >
+                      <span className={`absolute top-[1px] left-[1px] h-4 w-4 rounded-full bg-white shadow transition-transform ${
+                        config.channels.gotify?.enabled ? "translate-x-[18px]" : "translate-x-0"
+                      }`} />
+                    </button>
+                  </div>
+                  {config.channels.gotify?.enabled && (
+                    <>
+                      <div className="space-y-1.5">
+                        <Label className="text-[11px] text-muted-foreground">Server URL</Label>
+                        <Input
+                          className="h-7 text-xs font-mono"
+                          placeholder="https://gotify.example.com"
+                          value={config.channels.gotify?.url || ""}
+                          onChange={e => updateChannel("gotify", "url", e.target.value)}
+                          disabled={!editMode}
+                        />
+                      </div>
+                      <div className="space-y-1.5">
+                        <Label className="text-[11px] text-muted-foreground">App Token</Label>
+                        <div className="flex items-center gap-1.5">
+                          <Input
+                            type={showSecrets["gt_token"] ? "text" : "password"}
+                            className="h-7 text-xs font-mono"
+                            placeholder="A_valid_gotify_token"
+                            value={config.channels.gotify?.token || ""}
+                            onChange={e => updateChannel("gotify", "token", e.target.value)}
+                            disabled={!editMode}
+                          />
+                          <button
+                            className="h-7 w-7 flex items-center justify-center rounded-md border border-border hover:bg-muted transition-colors shrink-0"
+                            onClick={() => toggleSecret("gt_token")}
+                          >
+                            {showSecrets["gt_token"] ? <EyeOff className="h-3 w-3" /> : <Eye className="h-3 w-3" />}
+                          </button>
+                        </div>
+                      </div>
+                      {!editMode && config.channels.gotify?.url && (
+                        <button
+                          className="h-7 px-3 text-xs rounded-md border border-border bg-background hover:bg-muted transition-colors flex items-center gap-1.5 w-full justify-center"
+                          onClick={() => handleTest("gotify")}
+                          disabled={testing === "gotify"}
+                        >
+                          {testing === "gotify" ? (
+                            <Loader2 className="h-3 w-3 animate-spin" />
+                          ) : (
+                            <TestTube2 className="h-3 w-3" />
+                          )}
+                          Test Gotify
+                        </button>
+                      )}
+                    </>
+                  )}
+                </TabsContent>
+
+                {/* Discord */}
+                <TabsContent value="discord" className="space-y-3 pt-2">
+                  <div className="flex items-center justify-between">
+                    <Label className="text-xs font-medium">Enable Discord</Label>
+                    <button
+                      className={`relative w-9 h-[18px] rounded-full transition-colors ${
+                        config.channels.discord?.enabled ? "bg-indigo-600" : "bg-muted-foreground/30"
+                      } ${!editMode ? "opacity-60 cursor-not-allowed" : "cursor-pointer"}`}
+                      onClick={() => editMode && updateChannel("discord", "enabled", !config.channels.discord?.enabled)}
+                      disabled={!editMode}
+                      role="switch"
+                      aria-checked={config.channels.discord?.enabled || false}
+                    >
+                      <span className={`absolute top-[1px] left-[1px] h-4 w-4 rounded-full bg-white shadow transition-transform ${
+                        config.channels.discord?.enabled ? "translate-x-[18px]" : "translate-x-0"
+                      }`} />
+                    </button>
+                  </div>
+                  {config.channels.discord?.enabled && (
+                    <>
+                      <div className="space-y-1.5">
+                        <Label className="text-[11px] text-muted-foreground">Webhook URL</Label>
+                        <div className="flex items-center gap-1.5">
+                          <Input
+                            type={showSecrets["dc_hook"] ? "text" : "password"}
+                            className="h-7 text-xs font-mono"
+                            placeholder="https://discord.com/api/webhooks/..."
+                            value={config.channels.discord?.webhook_url || ""}
+                            onChange={e => updateChannel("discord", "webhook_url", e.target.value)}
+                            disabled={!editMode}
+                          />
+                          <button
+                            className="h-7 w-7 flex items-center justify-center rounded-md border border-border hover:bg-muted transition-colors shrink-0"
+                            onClick={() => toggleSecret("dc_hook")}
+                          >
+                            {showSecrets["dc_hook"] ? <EyeOff className="h-3 w-3" /> : <Eye className="h-3 w-3" />}
+                          </button>
+                        </div>
+                      </div>
+                      {!editMode && config.channels.discord?.webhook_url && (
+                        <button
+                          className="h-7 px-3 text-xs rounded-md border border-border bg-background hover:bg-muted transition-colors flex items-center gap-1.5 w-full justify-center"
+                          onClick={() => handleTest("discord")}
+                          disabled={testing === "discord"}
+                        >
+                          {testing === "discord" ? (
+                            <Loader2 className="h-3 w-3 animate-spin" />
+                          ) : (
+                            <TestTube2 className="h-3 w-3" />
+                          )}
+                          Test Discord
+                        </button>
+                      )}
+                    </>
+                  )}
+                </TabsContent>
+              </Tabs>
+
+              {/* Test Result */}
+              {testResult && (
+                <div className={`flex items-center gap-2 p-2.5 rounded-md text-xs ${
+                  testResult.success
+                    ? "bg-green-500/10 border border-green-500/20 text-green-400"
+                    : "bg-red-500/10 border border-red-500/20 text-red-400"
+                }`}>
+                  {testResult.success ? (
+                    <CheckCircle2 className="h-3.5 w-3.5 shrink-0" />
+                  ) : (
+                    <XCircle className="h-3.5 w-3.5 shrink-0" />
+                  )}
+                  <span>{testResult.message}</span>
+                </div>
+              )}
+            </div>
+
+            {/* ── Severity Filter ── */}
+            <div className="space-y-2">
+              <div className="flex items-center gap-2">
+                <AlertTriangle className="h-3.5 w-3.5 text-muted-foreground" />
+                <span className="text-xs font-medium text-muted-foreground uppercase tracking-wider">Severity Filter</span>
+              </div>
+              <Select
+                value={config.severity_filter}
+                onValueChange={v => updateConfig(p => ({ ...p, severity_filter: v }))}
+                disabled={!editMode}
+              >
+                <SelectTrigger className={`h-8 text-xs ${!editMode ? "opacity-60" : ""}`}>
+                  <SelectValue />
+                </SelectTrigger>
+                <SelectContent>
+                  {SEVERITY_OPTIONS.map(opt => (
+                    <SelectItem key={opt.value} value={opt.value}>{opt.label}</SelectItem>
+                  ))}
+                </SelectContent>
+              </Select>
+            </div>
+
+            {/* ── Event Categories ── */}
+            <div className="space-y-2">
+              <div className="flex items-center gap-2">
+                <Send className="h-3.5 w-3.5 text-muted-foreground" />
+                <span className="text-xs font-medium text-muted-foreground uppercase tracking-wider">Event Categories</span>
+              </div>
+              <div className="grid grid-cols-1 sm:grid-cols-2 gap-1.5">
+                {EVENT_CATEGORIES.map(cat => (
+                  <label
+                    key={cat.key}
+                    className={`flex items-start gap-2.5 p-2 rounded-md border border-border/50 transition-colors ${
+                      editMode ? "hover:bg-muted/50 cursor-pointer" : "opacity-60"
+                    }`}
+                  >
+                    <Checkbox
+                      checked={config.event_categories[cat.key] ?? true}
+                      onCheckedChange={checked => {
+                        if (!editMode) return
+                        updateConfig(p => ({
+                          ...p,
+                          event_categories: { ...p.event_categories, [cat.key]: !!checked },
+                        }))
+                      }}
+                      disabled={!editMode}
+                      className="mt-0.5"
+                    />
+                    <div className="min-w-0">
+                      <span className="text-xs font-medium block">{cat.label}</span>
+                      <span className="text-[10px] text-muted-foreground">{cat.desc}</span>
+                    </div>
+                  </label>
+                ))}
+              </div>
+            </div>
+
+            {/* ── Advanced: AI Enhancement ── */}
+            <div>
+              <button
+                className="flex items-center gap-2 text-xs text-muted-foreground hover:text-foreground transition-colors w-full py-1"
+                onClick={() => setShowAdvanced(!showAdvanced)}
+              >
+                {showAdvanced ? <ChevronUp className="h-3 w-3" /> : <ChevronDown className="h-3 w-3" />}
+                <span className="font-medium uppercase tracking-wider">Advanced: AI Enhancement</span>
+                {config.ai_enabled && (
+                  <Badge variant="outline" className="text-[9px] border-purple-500/30 text-purple-400 ml-1">
+                    ON
+                  </Badge>
+                )}
+              </button>
+
+              {showAdvanced && (
+                <div className="space-y-3 mt-3 p-3 rounded-lg bg-muted/30 border border-border/50">
+                  <div className="flex items-center justify-between">
+                    <div>
+                      <span className="text-xs font-medium">AI-Enhanced Messages</span>
+                      <p className="text-[10px] text-muted-foreground">Use AI to generate contextual notification messages</p>
+                    </div>
+                    <button
+                      className={`relative w-9 h-[18px] rounded-full transition-colors ${
+                        config.ai_enabled ? "bg-purple-600" : "bg-muted-foreground/30"
+                      } ${!editMode ? "opacity-60 cursor-not-allowed" : "cursor-pointer"}`}
+                      onClick={() => editMode && updateConfig(p => ({ ...p, ai_enabled: !p.ai_enabled }))}
+                      disabled={!editMode}
+                      role="switch"
+                      aria-checked={config.ai_enabled}
+                    >
+                      <span className={`absolute top-[1px] left-[1px] h-4 w-4 rounded-full bg-white shadow transition-transform ${
+                        config.ai_enabled ? "translate-x-[18px]" : "translate-x-0"
+                      }`} />
+                    </button>
+                  </div>
+
+                  {config.ai_enabled && (
+                    <>
+                      <div className="space-y-1.5">
+                        <Label className="text-[11px] text-muted-foreground">Provider</Label>
+                        <Select
+                          value={config.ai_provider}
+                          onValueChange={v => updateConfig(p => ({ ...p, ai_provider: v }))}
+                          disabled={!editMode}
+                        >
+                          <SelectTrigger className="h-7 text-xs">
+                            <SelectValue />
+                          </SelectTrigger>
+                          <SelectContent>
+                            {AI_PROVIDERS.map(p => (
+                              <SelectItem key={p.value} value={p.value}>{p.label}</SelectItem>
+                            ))}
+                          </SelectContent>
+                        </Select>
+                      </div>
+                      <div className="space-y-1.5">
+                        <Label className="text-[11px] text-muted-foreground">API Key</Label>
+                        <div className="flex items-center gap-1.5">
+                          <Input
+                            type={showSecrets["ai_key"] ? "text" : "password"}
+                            className="h-7 text-xs font-mono"
+                            placeholder="sk-..."
+                            value={config.ai_api_key}
+                            onChange={e => updateConfig(p => ({ ...p, ai_api_key: e.target.value }))}
+                            disabled={!editMode}
+                          />
+                          <button
+                            className="h-7 w-7 flex items-center justify-center rounded-md border border-border hover:bg-muted transition-colors shrink-0"
+                            onClick={() => toggleSecret("ai_key")}
+                          >
+                            {showSecrets["ai_key"] ? <EyeOff className="h-3 w-3" /> : <Eye className="h-3 w-3" />}
+                          </button>
+                        </div>
+                      </div>
+                      <div className="space-y-1.5">
+                        <Label className="text-[11px] text-muted-foreground">Model (optional)</Label>
+                        <Input
+                          className="h-7 text-xs font-mono"
+                          placeholder={config.ai_provider === "openai" ? "gpt-4o-mini" : "llama-3.3-70b-versatile"}
+                          value={config.ai_model}
+                          onChange={e => updateConfig(p => ({ ...p, ai_model: e.target.value }))}
+                          disabled={!editMode}
+                        />
+                      </div>
+                      <div className="flex items-start gap-2 p-2 rounded-md bg-purple-500/10 border border-purple-500/20">
+                        <Info className="h-3.5 w-3.5 text-purple-400 shrink-0 mt-0.5" />
+                        <p className="text-[10px] text-purple-400/90 leading-relaxed">
+                          AI enhancement is optional. When enabled, notifications include contextual analysis and recommended actions. If the AI service is unavailable, standard templates are used as fallback.
+                        </p>
+                      </div>
+                    </>
+                  )}
+                </div>
+              )}
+            </div>
+
+            {/* ── Notification History ── */}
+            <div>
+              <button
+                className="flex items-center gap-2 text-xs text-muted-foreground hover:text-foreground transition-colors w-full py-1"
+                onClick={() => setShowHistory(!showHistory)}
+              >
+                {showHistory ? <ChevronUp className="h-3 w-3" /> : <ChevronDown className="h-3 w-3" />}
+                <span className="font-medium uppercase tracking-wider">Recent History</span>
+                {history.length > 0 && (
+                  <Badge variant="outline" className="text-[9px] ml-1">{history.length}</Badge>
+                )}
+              </button>
+
+              {showHistory && (
+                <div className="mt-3 space-y-2">
+                  {history.length === 0 ? (
+                    <p className="text-xs text-muted-foreground text-center py-4">No notifications sent yet</p>
+                  ) : (
+                    <>
+                      <div className="flex items-center justify-end">
+                        <button
+                          className="h-6 px-2 text-[10px] rounded-md text-muted-foreground hover:text-red-400 transition-colors flex items-center gap-1"
+                          onClick={handleClearHistory}
+                        >
+                          <Trash2 className="h-3 w-3" />
+                          Clear
+                        </button>
+                      </div>
+                      <div className="space-y-1 max-h-48 overflow-y-auto">
+                        {history.map(entry => (
+                          <div
+                            key={entry.id}
+                            className="flex items-center gap-2 p-2 rounded-md bg-muted/30 border border-border/50"
+                          >
+                            {entry.success ? (
+                              <CheckCircle2 className="h-3 w-3 text-green-500 shrink-0" />
+                            ) : (
+                              <XCircle className="h-3 w-3 text-red-500 shrink-0" />
+                            )}
+                            <div className="flex-1 min-w-0">
+                              <span className="text-[11px] font-medium truncate block">{entry.title || entry.event_type}</span>
+                              <span className="text-[10px] text-muted-foreground">
+                                {entry.channel} - {new Date(entry.sent_at).toLocaleString()}
+                              </span>
+                            </div>
+                            <Badge
+                              variant="outline"
+                              className={`text-[9px] shrink-0 ${
+                                entry.severity === "critical"
+                                  ? "border-red-500/30 text-red-400"
+                                  : entry.severity === "warning"
+                                  ? "border-amber-500/30 text-amber-400"
+                                  : "border-blue-500/30 text-blue-400"
+                              }`}
+                            >
+                              {entry.severity}
+                            </Badge>
+                          </div>
+                        ))}
+                      </div>
+                    </>
+                  )}
+                </div>
+              )}
+            </div>
+          </>
+        )}
+
+        {/* ── Footer info ── */}
+        <div className="flex items-start gap-2 pt-3 border-t border-border">
+          <Info className="h-3.5 w-3.5 text-blue-400 shrink-0 mt-0.5" />
+          <p className="text-[11px] text-muted-foreground leading-relaxed">
+            {config.enabled
+              ? "Notifications are active. Events matching your severity filter and category selection will be sent to configured channels."
+              : "Enable notifications to receive alerts about system events, health status changes, and security incidents via Telegram, Gotify, or Discord."}
+          </p>
+        </div>
+      </CardContent>
+    </Card>
+  )
+}

AppImage/components/settings.tsx

@@ -3,6 +3,7 @@
 import { useState, useEffect } from "react"
 import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "./ui/card"
 import { Wrench, Package, Ruler, HeartPulse, Cpu, MemoryStick, HardDrive, CircleDot, Network, Server, Settings2, FileText, RefreshCw, Shield, AlertTriangle, Info, Loader2, Check } from "lucide-react"
+import { NotificationSettings } from "./notification-settings"
 import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "./ui/select"
 import { Input } from "./ui/input"
 import { Badge } from "./ui/badge"
@@ -438,6 +439,9 @@ export function Settings() {
         </CardContent>
       </Card>
 
+      {/* Notification Settings */}
+      <NotificationSettings />
+
       {/* ProxMenux Optimizations */}
       <Card>
         <CardHeader>

AppImage/scripts/build_appimage.sh

@@ -91,6 +91,11 @@ cp "$SCRIPT_DIR/proxmox_storage_monitor.py" "$APP_DIR/usr/bin/" 2>/dev/null || e
 cp "$SCRIPT_DIR/flask_script_runner.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️  flask_script_runner.py not found"
 cp "$SCRIPT_DIR/security_manager.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️  security_manager.py not found"
 cp "$SCRIPT_DIR/flask_security_routes.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️  flask_security_routes.py not found"
+cp "$SCRIPT_DIR/notification_manager.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️  notification_manager.py not found"
+cp "$SCRIPT_DIR/notification_channels.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️  notification_channels.py not found"
+cp "$SCRIPT_DIR/notification_templates.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️  notification_templates.py not found"
+cp "$SCRIPT_DIR/notification_events.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️  notification_events.py not found"
+cp "$SCRIPT_DIR/flask_notification_routes.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️  flask_notification_routes.py not found"
 
 echo "📋 Adding translation support..."
 cat > "$APP_DIR/usr/bin/translate_cli.py" << 'PYEOF'

AppImage/scripts/flask_notification_routes.py

@@ -0,0 +1,102 @@
+"""
+Flask routes for notification service configuration and management.
+Blueprint pattern matching flask_health_routes.py / flask_security_routes.py.
+"""
+
+from flask import Blueprint, jsonify, request
+from notification_manager import notification_manager
+
+notification_bp = Blueprint('notifications', __name__)
+
+
+@notification_bp.route('/api/notifications/settings', methods=['GET'])
+def get_notification_settings():
+    """Get all notification settings for the UI."""
+    try:
+        settings = notification_manager.get_settings()
+        return jsonify(settings)
+    except Exception as e:
+        return jsonify({'error': str(e)}), 500
+
+
+@notification_bp.route('/api/notifications/settings', methods=['POST'])
+def save_notification_settings():
+    """Save notification settings from the UI."""
+    try:
+        payload = request.get_json()
+        if not payload:
+            return jsonify({'error': 'No data provided'}), 400
+        
+        result = notification_manager.save_settings(payload)
+        return jsonify(result)
+    except Exception as e:
+        return jsonify({'error': str(e)}), 500
+
+
+@notification_bp.route('/api/notifications/test', methods=['POST'])
+def test_notification():
+    """Send a test notification to one or all channels."""
+    try:
+        data = request.get_json() or {}
+        channel = data.get('channel', 'all')
+        
+        result = notification_manager.test_channel(channel)
+        return jsonify(result)
+    except Exception as e:
+        return jsonify({'error': str(e)}), 500
+
+
+@notification_bp.route('/api/notifications/status', methods=['GET'])
+def get_notification_status():
+    """Get notification service status."""
+    try:
+        status = notification_manager.get_status()
+        return jsonify(status)
+    except Exception as e:
+        return jsonify({'error': str(e)}), 500
+
+
+@notification_bp.route('/api/notifications/history', methods=['GET'])
+def get_notification_history():
+    """Get notification history with optional filters."""
+    try:
+        limit = request.args.get('limit', 50, type=int)
+        offset = request.args.get('offset', 0, type=int)
+        severity = request.args.get('severity', '')
+        channel = request.args.get('channel', '')
+        
+        result = notification_manager.get_history(limit, offset, severity, channel)
+        return jsonify(result)
+    except Exception as e:
+        return jsonify({'error': str(e)}), 500
+
+
+@notification_bp.route('/api/notifications/history', methods=['DELETE'])
+def clear_notification_history():
+    """Clear all notification history."""
+    try:
+        result = notification_manager.clear_history()
+        return jsonify(result)
+    except Exception as e:
+        return jsonify({'error': str(e)}), 500
+
+
+@notification_bp.route('/api/notifications/send', methods=['POST'])
+def send_notification():
+    """Send a notification via API (for testing or external triggers)."""
+    try:
+        data = request.get_json()
+        if not data:
+            return jsonify({'error': 'No data provided'}), 400
+        
+        result = notification_manager.send_notification(
+            event_type=data.get('event_type', 'custom'),
+            severity=data.get('severity', 'INFO'),
+            title=data.get('title', ''),
+            message=data.get('message', ''),
+            data=data.get('data', {}),
+            source='api'
+        )
+        return jsonify(result)
+    except Exception as e:
+        return jsonify({'error': str(e)}), 500

AppImage/scripts/flask_server.py

@@ -46,6 +46,8 @@ from flask_health_routes import health_bp  # noqa: E402
 from flask_auth_routes import auth_bp  # noqa: E402
 from flask_proxmenux_routes import proxmenux_bp  # noqa: E402
 from flask_security_routes import security_bp  # noqa: E402
+from flask_notification_routes import notification_bp  # noqa: E402
+from notification_manager import notification_manager  # noqa: E402
 from jwt_middleware import require_auth  # noqa: E402
 import auth_manager  # noqa: E402
 
@@ -120,6 +122,7 @@ app.register_blueprint(auth_bp)
 app.register_blueprint(health_bp)
 app.register_blueprint(proxmenux_bp)
 app.register_blueprint(security_bp)
+app.register_blueprint(notification_bp)
 
 # Initialize terminal / WebSocket routes
 init_terminal_routes(app)
@@ -7094,6 +7097,16 @@ if __name__ == '__main__':
     except Exception as e:
         print(f"[ProxMenux] Vital signs sampler failed to start: {e}")
 
+    # ── Notification Service ──
+    try:
+        notification_manager.start()
+        if notification_manager._enabled:
+            print(f"[ProxMenux] Notification service started (channels: {list(notification_manager._channels.keys())})")
+        else:
+            print("[ProxMenux] Notification service loaded (disabled - configure in Settings)")
+    except Exception as e:
+        print(f"[ProxMenux] Notification service failed to start: {e}")
+
     # Check for SSL configuration
     ssl_ctx = None
     try:

AppImage/scripts/health_persistence.py

@@ -114,6 +114,22 @@ class HealthPersistence:
             )
         ''')
         
+        # Notification history table (records all sent notifications)
+        cursor.execute('''
+            CREATE TABLE IF NOT EXISTS notification_history (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                event_type TEXT NOT NULL,
+                channel TEXT NOT NULL,
+                title TEXT,
+                message TEXT,
+                severity TEXT,
+                sent_at TEXT NOT NULL,
+                success INTEGER DEFAULT 1,
+                error_message TEXT,
+                source TEXT DEFAULT 'server'
+            )
+        ''')
+        
         # Migration: add suppression_hours column to errors if not present
         cursor.execute("PRAGMA table_info(errors)")
         columns = [col[1] for col in cursor.fetchall()]
@@ -125,6 +141,8 @@ class HealthPersistence:
         cursor.execute('CREATE INDEX IF NOT EXISTS idx_category ON errors(category)')
         cursor.execute('CREATE INDEX IF NOT EXISTS idx_resolved ON errors(resolved_at)')
         cursor.execute('CREATE INDEX IF NOT EXISTS idx_events_error ON events(error_key)')
+        cursor.execute('CREATE INDEX IF NOT EXISTS idx_notif_sent_at ON notification_history(sent_at)')
+        cursor.execute('CREATE INDEX IF NOT EXISTS idx_notif_severity ON notification_history(severity)')
         
         conn.commit()
         conn.close()

AppImage/scripts/notification_channels.py

@@ -0,0 +1,402 @@
+"""
+ProxMenux Notification Channels
+Provides transport adapters for Telegram, Gotify, and Discord.
+
+Each channel implements send() and test() with:
+- Retry with exponential backoff (3 attempts)
+- Request timeout of 10s
+- Rate limiting (max 30 msg/min per channel)
+
+Author: MacRimi
+"""
+
+import json
+import time
+import urllib.request
+import urllib.error
+import urllib.parse
+from abc import ABC, abstractmethod
+from collections import deque
+from typing import Tuple, Optional, Dict, Any
+
+
+# ─── Rate Limiter ────────────────────────────────────────────────
+
+class RateLimiter:
+    """Token-bucket rate limiter: max N messages per window."""
+    
+    def __init__(self, max_calls: int = 30, window_seconds: int = 60):
+        self.max_calls = max_calls
+        self.window = window_seconds
+        self._timestamps: deque = deque()
+    
+    def allow(self) -> bool:
+        now = time.monotonic()
+        while self._timestamps and now - self._timestamps[0] > self.window:
+            self._timestamps.popleft()
+        if len(self._timestamps) >= self.max_calls:
+            return False
+        self._timestamps.append(now)
+        return True
+    
+    def wait_time(self) -> float:
+        if not self._timestamps:
+            return 0.0
+        return max(0.0, self.window - (time.monotonic() - self._timestamps[0]))
+
+
+# ─── Base Channel ────────────────────────────────────────────────
+
+class NotificationChannel(ABC):
+    """Abstract base for all notification channels."""
+    
+    MAX_RETRIES = 3
+    RETRY_DELAYS = [2, 4, 8]  # exponential backoff seconds
+    REQUEST_TIMEOUT = 10
+    
+    def __init__(self):
+        self._rate_limiter = RateLimiter(max_calls=30, window_seconds=60)
+    
+    @abstractmethod
+    def send(self, title: str, message: str, severity: str = 'INFO',
+             data: Optional[Dict] = None) -> Dict[str, Any]:
+        """Send a notification. Returns {success, error, channel}."""
+        pass
+    
+    @abstractmethod
+    def test(self) -> Tuple[bool, str]:
+        """Send a test message. Returns (success, error_message)."""
+        pass
+    
+    @abstractmethod
+    def validate_config(self) -> Tuple[bool, str]:
+        """Check if config is valid without sending. Returns (valid, error)."""
+        pass
+    
+    def _http_request(self, url: str, data: bytes, headers: Dict[str, str],
+                      method: str = 'POST') -> Tuple[int, str]:
+        """Execute HTTP request with timeout. Returns (status_code, body)."""
+        req = urllib.request.Request(url, data=data, headers=headers, method=method)
+        try:
+            with urllib.request.urlopen(req, timeout=self.REQUEST_TIMEOUT) as resp:
+                body = resp.read().decode('utf-8', errors='replace')
+                return resp.status, body
+        except urllib.error.HTTPError as e:
+            body = e.read().decode('utf-8', errors='replace') if e.fp else str(e)
+            return e.code, body
+        except urllib.error.URLError as e:
+            return 0, str(e.reason)
+        except Exception as e:
+            return 0, str(e)
+    
+    def _send_with_retry(self, send_fn) -> Dict[str, Any]:
+        """Wrap a send function with rate limiting and retry logic."""
+        if not self._rate_limiter.allow():
+            wait = self._rate_limiter.wait_time()
+            return {
+                'success': False,
+                'error': f'Rate limited. Retry in {wait:.0f}s',
+                'rate_limited': True
+            }
+        
+        last_error = ''
+        for attempt in range(self.MAX_RETRIES):
+            try:
+                status, body = send_fn()
+                if 200 <= status < 300:
+                    return {'success': True, 'error': None}
+                last_error = f'HTTP {status}: {body[:200]}'
+            except Exception as e:
+                last_error = str(e)
+            
+            if attempt < self.MAX_RETRIES - 1:
+                time.sleep(self.RETRY_DELAYS[attempt])
+        
+        return {'success': False, 'error': last_error}
+
+
+# ─── Telegram ────────────────────────────────────────────────────
+
+class TelegramChannel(NotificationChannel):
+    """Telegram Bot API channel using HTML parse mode."""
+    
+    API_BASE = 'https://api.telegram.org/bot{token}/sendMessage'
+    MAX_LENGTH = 4096
+    
+    SEVERITY_ICONS = {
+        'CRITICAL': '\U0001F534',  # red circle
+        'WARNING':  '\U0001F7E1',  # yellow circle
+        'INFO':     '\U0001F535',  # blue circle
+        'OK':       '\U0001F7E2',  # green circle
+        'UNKNOWN':  '\u26AA',      # white circle
+    }
+    
+    def __init__(self, bot_token: str, chat_id: str):
+        super().__init__()
+        self.bot_token = bot_token.strip()
+        self.chat_id = chat_id.strip()
+    
+    def validate_config(self) -> Tuple[bool, str]:
+        if not self.bot_token:
+            return False, 'Bot token is required'
+        if not self.chat_id:
+            return False, 'Chat ID is required'
+        if ':' not in self.bot_token:
+            return False, 'Invalid bot token format (expected BOT_ID:TOKEN)'
+        return True, ''
+    
+    def send(self, title: str, message: str, severity: str = 'INFO',
+             data: Optional[Dict] = None) -> Dict[str, Any]:
+        icon = self.SEVERITY_ICONS.get(severity, self.SEVERITY_ICONS['INFO'])
+        html_msg = f"<b>{icon} {self._escape_html(title)}</b>\n\n{self._escape_html(message)}"
+        
+        # Split long messages
+        chunks = self._split_message(html_msg)
+        result = {'success': True, 'error': None, 'channel': 'telegram'}
+        
+        for chunk in chunks:
+            res = self._send_with_retry(lambda c=chunk: self._post_message(c))
+            if not res['success']:
+                result = {**res, 'channel': 'telegram'}
+                break
+        
+        return result
+    
+    def test(self) -> Tuple[bool, str]:
+        valid, err = self.validate_config()
+        if not valid:
+            return False, err
+        
+        result = self.send(
+            'ProxMenux Test',
+            'Notification service is working correctly.\nThis is a test message from ProxMenux Monitor.',
+            'INFO'
+        )
+        return result['success'], result.get('error', '')
+    
+    def _post_message(self, text: str) -> Tuple[int, str]:
+        url = self.API_BASE.format(token=self.bot_token)
+        payload = json.dumps({
+            'chat_id': self.chat_id,
+            'text': text,
+            'parse_mode': 'HTML',
+            'disable_web_page_preview': True,
+        }).encode('utf-8')
+        
+        return self._http_request(url, payload, {'Content-Type': 'application/json'})
+    
+    def _split_message(self, text: str) -> list:
+        if len(text) <= self.MAX_LENGTH:
+            return [text]
+        chunks = []
+        while text:
+            if len(text) <= self.MAX_LENGTH:
+                chunks.append(text)
+                break
+            split_at = text.rfind('\n', 0, self.MAX_LENGTH)
+            if split_at == -1:
+                split_at = self.MAX_LENGTH
+            chunks.append(text[:split_at])
+            text = text[split_at:].lstrip('\n')
+        return chunks
+    
+    @staticmethod
+    def _escape_html(text: str) -> str:
+        return (text
+                .replace('&', '&amp;')
+                .replace('<', '&lt;')
+                .replace('>', '&gt;'))
+
+
+# ─── Gotify ──────────────────────────────────────────────────────
+
+class GotifyChannel(NotificationChannel):
+    """Gotify push notification channel with priority mapping."""
+    
+    PRIORITY_MAP = {
+        'OK':       1,
+        'INFO':     2,
+        'UNKNOWN':  3,
+        'WARNING':  5,
+        'CRITICAL': 10,
+    }
+    
+    def __init__(self, server_url: str, app_token: str):
+        super().__init__()
+        self.server_url = server_url.rstrip('/').strip()
+        self.app_token = app_token.strip()
+    
+    def validate_config(self) -> Tuple[bool, str]:
+        if not self.server_url:
+            return False, 'Server URL is required'
+        if not self.app_token:
+            return False, 'Application token is required'
+        if not self.server_url.startswith(('http://', 'https://')):
+            return False, 'Server URL must start with http:// or https://'
+        return True, ''
+    
+    def send(self, title: str, message: str, severity: str = 'INFO',
+             data: Optional[Dict] = None) -> Dict[str, Any]:
+        priority = self.PRIORITY_MAP.get(severity, 2)
+        
+        result = self._send_with_retry(
+            lambda: self._post_message(title, message, priority)
+        )
+        result['channel'] = 'gotify'
+        return result
+    
+    def test(self) -> Tuple[bool, str]:
+        valid, err = self.validate_config()
+        if not valid:
+            return False, err
+        
+        result = self.send(
+            'ProxMenux Test',
+            'Notification service is working correctly.\nThis is a test message from ProxMenux Monitor.',
+            'INFO'
+        )
+        return result['success'], result.get('error', '')
+    
+    def _post_message(self, title: str, message: str, priority: int) -> Tuple[int, str]:
+        url = f"{self.server_url}/message?token={self.app_token}"
+        payload = json.dumps({
+            'title': title,
+            'message': message,
+            'priority': priority,
+            'extras': {
+                'client::display': {'contentType': 'text/markdown'}
+            }
+        }).encode('utf-8')
+        
+        return self._http_request(url, payload, {'Content-Type': 'application/json'})
+
+
+# ─── Discord ─────────────────────────────────────────────────────
+
+class DiscordChannel(NotificationChannel):
+    """Discord webhook channel with color-coded embeds."""
+    
+    MAX_EMBED_DESC = 2048
+    
+    SEVERITY_COLORS = {
+        'CRITICAL': 0xED4245,   # red
+        'WARNING':  0xFEE75C,   # yellow
+        'INFO':     0x5865F2,   # blurple
+        'OK':       0x57F287,   # green
+        'UNKNOWN':  0x99AAB5,   # grey
+    }
+    
+    def __init__(self, webhook_url: str):
+        super().__init__()
+        self.webhook_url = webhook_url.strip()
+    
+    def validate_config(self) -> Tuple[bool, str]:
+        if not self.webhook_url:
+            return False, 'Webhook URL is required'
+        if 'discord.com/api/webhooks/' not in self.webhook_url:
+            return False, 'Invalid Discord webhook URL'
+        return True, ''
+    
+    def send(self, title: str, message: str, severity: str = 'INFO',
+             data: Optional[Dict] = None) -> Dict[str, Any]:
+        color = self.SEVERITY_COLORS.get(severity, 0x5865F2)
+        
+        desc = message[:self.MAX_EMBED_DESC] if len(message) > self.MAX_EMBED_DESC else message
+        
+        embed = {
+            'title': title,
+            'description': desc,
+            'color': color,
+            'footer': {'text': 'ProxMenux Monitor'},
+            'timestamp': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
+        }
+        
+        if data:
+            fields = []
+            if data.get('category'):
+                fields.append({'name': 'Category', 'value': data['category'], 'inline': True})
+            if data.get('hostname'):
+                fields.append({'name': 'Host', 'value': data['hostname'], 'inline': True})
+            if data.get('severity'):
+                fields.append({'name': 'Severity', 'value': data['severity'], 'inline': True})
+            if fields:
+                embed['fields'] = fields
+        
+        result = self._send_with_retry(
+            lambda: self._post_webhook(embed)
+        )
+        result['channel'] = 'discord'
+        return result
+    
+    def test(self) -> Tuple[bool, str]:
+        valid, err = self.validate_config()
+        if not valid:
+            return False, err
+        
+        result = self.send(
+            'ProxMenux Test',
+            'Notification service is working correctly.\nThis is a test message from ProxMenux Monitor.',
+            'INFO'
+        )
+        return result['success'], result.get('error', '')
+    
+    def _post_webhook(self, embed: Dict) -> Tuple[int, str]:
+        payload = json.dumps({
+            'username': 'ProxMenux',
+            'embeds': [embed]
+        }).encode('utf-8')
+        
+        return self._http_request(
+            self.webhook_url, payload, {'Content-Type': 'application/json'}
+        )
+
+
+# ─── Channel Factory ─────────────────────────────────────────────
+
+CHANNEL_TYPES = {
+    'telegram': {
+        'name': 'Telegram',
+        'config_keys': ['bot_token', 'chat_id'],
+        'class': TelegramChannel,
+    },
+    'gotify': {
+        'name': 'Gotify',
+        'config_keys': ['url', 'token'],
+        'class': GotifyChannel,
+    },
+    'discord': {
+        'name': 'Discord',
+        'config_keys': ['webhook_url'],
+        'class': DiscordChannel,
+    },
+}
+
+
+def create_channel(channel_type: str, config: Dict[str, str]) -> Optional[NotificationChannel]:
+    """Create a channel instance from type name and config dict.
+    
+    Args:
+        channel_type: 'telegram', 'gotify', or 'discord'
+        config: Dict with channel-specific keys (see CHANNEL_TYPES)
+    
+    Returns:
+        Channel instance or None if creation fails
+    """
+    try:
+        if channel_type == 'telegram':
+            return TelegramChannel(
+                bot_token=config.get('bot_token', ''),
+                chat_id=config.get('chat_id', '')
+            )
+        elif channel_type == 'gotify':
+            return GotifyChannel(
+                server_url=config.get('url', ''),
+                app_token=config.get('token', '')
+            )
+        elif channel_type == 'discord':
+            return DiscordChannel(
+                webhook_url=config.get('webhook_url', '')
+            )
+    except Exception as e:
+        print(f"[NotificationChannels] Failed to create {channel_type}: {e}")
+    return None

AppImage/scripts/notification_events.py

@@ -0,0 +1,698 @@
+"""
+ProxMenux Notification Event Watchers
+Detects Proxmox events from journald, PVE task log, and health monitor.
+
+Architecture:
+- JournalWatcher: Real-time stream of journald for critical events
+- TaskWatcher: Real-time tail of /var/log/pve/tasks/index for VM/CT/backup events
+- PollingCollector: Periodic poll of health_persistence pending notifications
+
+All watchers put events into a shared Queue consumed by NotificationManager.
+
+Author: MacRimi
+"""
+
+import os
+import re
+import json
+import time
+import socket
+import subprocess
+import threading
+from queue import Queue
+from typing import Optional, Dict, Any
+from pathlib import Path
+
+
+# ─── Event Object ─────────────────────────────────────────────────
+
+class NotificationEvent:
+    """Represents a detected event ready for notification dispatch."""
+    
+    __slots__ = ('event_type', 'severity', 'data', 'timestamp', 'source')
+    
+    def __init__(self, event_type: str, severity: str = 'INFO',
+                 data: Optional[Dict[str, Any]] = None,
+                 source: str = 'watcher'):
+        self.event_type = event_type
+        self.severity = severity
+        self.data = data or {}
+        self.timestamp = time.time()
+        self.source = source
+    
+    def __repr__(self):
+        return f"NotificationEvent({self.event_type}, {self.severity})"
+
+
+def _hostname() -> str:
+    try:
+        return socket.gethostname().split('.')[0]
+    except Exception:
+        return 'proxmox'
+
+
+# ─── Journal Watcher (Real-time) ─────────────────────────────────
+
+class JournalWatcher:
+    """Watches journald in real-time for critical system events.
+    
+    Uses 'journalctl -f -o json' subprocess to stream entries.
+    Detects: auth failures, kernel panics, OOM, service crashes,
+    disk I/O errors, split-brain, node disconnect, system shutdown,
+    fail2ban bans, firewall blocks, permission changes.
+    """
+    
+    def __init__(self, event_queue: Queue):
+        self._queue = event_queue
+        self._running = False
+        self._thread: Optional[threading.Thread] = None
+        self._process: Optional[subprocess.Popen] = None
+        self._hostname = _hostname()
+        
+        # Dedup: track recent events to avoid duplicates
+        self._recent_events: Dict[str, float] = {}
+        self._dedup_window = 30  # seconds
+    
+    def start(self):
+        """Start the journal watcher thread."""
+        if self._running:
+            return
+        self._running = True
+        self._thread = threading.Thread(target=self._watch_loop, daemon=True,
+                                        name='journal-watcher')
+        self._thread.start()
+    
+    def stop(self):
+        """Stop the journal watcher."""
+        self._running = False
+        if self._process:
+            try:
+                self._process.terminate()
+                self._process.wait(timeout=5)
+            except Exception:
+                try:
+                    self._process.kill()
+                except Exception:
+                    pass
+    
+    def _watch_loop(self):
+        """Main watch loop with auto-restart on failure."""
+        while self._running:
+            try:
+                self._run_journalctl()
+            except Exception as e:
+                print(f"[JournalWatcher] Error: {e}")
+            if self._running:
+                time.sleep(5)  # Wait before restart
+    
+    def _run_journalctl(self):
+        """Run journalctl -f and process output line by line."""
+        cmd = ['journalctl', '-f', '-o', 'json', '--no-pager',
+               '-n', '0']  # Start from now, don't replay history
+        
+        self._process = subprocess.Popen(
+            cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
+            text=True, bufsize=1
+        )
+        
+        for line in self._process.stdout:
+            if not self._running:
+                break
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                entry = json.loads(line)
+                self._process_entry(entry)
+            except (json.JSONDecodeError, KeyError):
+                # Try plain text matching as fallback
+                self._process_plain(line)
+        
+        if self._process:
+            self._process.wait()
+    
+    def _process_entry(self, entry: Dict):
+        """Process a parsed journald JSON entry."""
+        msg = entry.get('MESSAGE', '')
+        if not msg or not isinstance(msg, str):
+            return
+        
+        unit = entry.get('_SYSTEMD_UNIT', '')
+        syslog_id = entry.get('SYSLOG_IDENTIFIER', '')
+        priority = int(entry.get('PRIORITY', 6))
+        
+        self._check_auth_failure(msg, syslog_id, entry)
+        self._check_fail2ban(msg, syslog_id)
+        self._check_kernel_critical(msg, syslog_id, priority)
+        self._check_service_failure(msg, unit)
+        self._check_disk_io(msg, syslog_id, priority)
+        self._check_cluster_events(msg, syslog_id)
+        self._check_system_shutdown(msg, syslog_id)
+        self._check_permission_change(msg, syslog_id)
+        self._check_firewall(msg, syslog_id)
+    
+    def _process_plain(self, line: str):
+        """Fallback: process a plain text log line."""
+        self._check_auth_failure(line, '', {})
+        self._check_fail2ban(line, '')
+        self._check_kernel_critical(line, '', 6)
+        self._check_cluster_events(line, '')
+        self._check_system_shutdown(line, '')
+    
+    # ── Detection methods ──
+    
+    def _check_auth_failure(self, msg: str, syslog_id: str, entry: Dict):
+        """Detect authentication failures (SSH, PAM, PVE)."""
+        patterns = [
+            (r'Failed password for (?:invalid user )?(\S+) from (\S+)', 'ssh'),
+            (r'authentication failure.*rhost=(\S+).*user=(\S+)', 'pam'),
+            (r'pvedaemon\[.*authentication failure.*rhost=(\S+)', 'pve'),
+        ]
+        
+        for pattern, service in patterns:
+            match = re.search(pattern, msg, re.IGNORECASE)
+            if match:
+                groups = match.groups()
+                if service == 'ssh':
+                    username, source_ip = groups[0], groups[1]
+                elif service == 'pam':
+                    source_ip, username = groups[0], groups[1]
+                else:
+                    source_ip = groups[0]
+                    username = 'unknown'
+                
+                self._emit('auth_fail', 'WARNING', {
+                    'source_ip': source_ip,
+                    'username': username,
+                    'service': service,
+                    'hostname': self._hostname,
+                })
+                return
+    
+    def _check_fail2ban(self, msg: str, syslog_id: str):
+        """Detect Fail2Ban IP bans."""
+        if 'fail2ban' not in msg.lower() and syslog_id != 'fail2ban-server':
+            return
+        
+        # Ban detected
+        ban_match = re.search(r'Ban\s+(\S+)', msg)
+        if ban_match:
+            ip = ban_match.group(1)
+            jail_match = re.search(r'\[(\w+)\]', msg)
+            jail = jail_match.group(1) if jail_match else 'unknown'
+            
+            self._emit('ip_block', 'INFO', {
+                'source_ip': ip,
+                'jail': jail,
+                'failures': '',
+                'hostname': self._hostname,
+            })
+    
+    def _check_kernel_critical(self, msg: str, syslog_id: str, priority: int):
+        """Detect kernel panics, OOM, segfaults, hardware errors."""
+        critical_patterns = {
+            r'kernel panic':       ('system_problem', 'CRITICAL', 'Kernel panic'),
+            r'Out of memory':      ('system_problem', 'CRITICAL', 'Out of memory killer activated'),
+            r'segfault':           ('system_problem', 'WARNING',  'Segmentation fault detected'),
+            r'BUG:':               ('system_problem', 'CRITICAL', 'Kernel BUG detected'),
+            r'Call Trace:':        ('system_problem', 'WARNING',  'Kernel call trace'),
+            r'I/O error.*dev\s+(\S+)': ('disk_io_error', 'CRITICAL', 'Disk I/O error'),
+            r'EXT4-fs error':      ('disk_io_error', 'CRITICAL', 'Filesystem error'),
+            r'BTRFS error':        ('disk_io_error', 'CRITICAL', 'Filesystem error'),
+            r'XFS.*error':         ('disk_io_error', 'CRITICAL', 'Filesystem error'),
+            r'ZFS.*error':         ('disk_io_error', 'CRITICAL', 'ZFS pool error'),
+            r'mce:.*Hardware Error': ('system_problem', 'CRITICAL', 'Hardware error (MCE)'),
+        }
+        
+        for pattern, (event_type, severity, reason) in critical_patterns.items():
+            if re.search(pattern, msg, re.IGNORECASE):
+                data = {'reason': reason, 'hostname': self._hostname}
+                
+                # Try to extract device for disk errors
+                dev_match = re.search(r'dev\s+(\S+)', msg)
+                if dev_match and event_type == 'disk_io_error':
+                    data['device'] = dev_match.group(1)
+                
+                self._emit(event_type, severity, data)
+                return
+    
+    def _check_service_failure(self, msg: str, unit: str):
+        """Detect critical service failures."""
+        service_patterns = [
+            r'Failed to start (.+)',
+            r'Unit (\S+) (?:entered failed state|failed)',
+            r'(\S+)\.service: (?:Main process exited|Failed with result)',
+        ]
+        
+        for pattern in service_patterns:
+            match = re.search(pattern, msg)
+            if match:
+                service_name = match.group(1)
+                self._emit('service_fail', 'WARNING', {
+                    'service_name': service_name,
+                    'reason': msg[:200],
+                    'hostname': self._hostname,
+                })
+                return
+    
+    def _check_disk_io(self, msg: str, syslog_id: str, priority: int):
+        """Detect disk I/O errors from kernel messages."""
+        if syslog_id != 'kernel' and priority > 3:
+            return
+        
+        io_patterns = [
+            r'blk_update_request: I/O error.*dev (\S+)',
+            r'Buffer I/O error on device (\S+)',
+            r'SCSI error.*sd(\w)',
+            r'ata\d+.*error',
+        ]
+        
+        for pattern in io_patterns:
+            match = re.search(pattern, msg)
+            if match:
+                device = match.group(1) if match.lastindex else 'unknown'
+                self._emit('disk_io_error', 'CRITICAL', {
+                    'device': device,
+                    'reason': msg[:200],
+                    'hostname': self._hostname,
+                })
+                return
+    
+    def _check_cluster_events(self, msg: str, syslog_id: str):
+        """Detect cluster split-brain and node disconnect."""
+        msg_lower = msg.lower()
+        
+        # Split-brain
+        if any(p in msg_lower for p in ['split-brain', 'split brain',
+                                          'fencing required', 'cluster partition']):
+            quorum = 'unknown'
+            if 'quorum' in msg_lower:
+                quorum = 'lost' if 'lost' in msg_lower else 'valid'
+            
+            self._emit('split_brain', 'CRITICAL', {
+                'quorum': quorum,
+                'reason': msg[:200],
+                'hostname': self._hostname,
+            })
+            return
+        
+        # Node disconnect
+        if (('quorum' in msg_lower and 'lost' in msg_lower) or
+            ('node' in msg_lower and any(w in msg_lower for w in ['left', 'offline', 'lost']))):
+            
+            node_match = re.search(r'[Nn]ode\s+(\S+)', msg)
+            node_name = node_match.group(1) if node_match else 'unknown'
+            
+            self._emit('node_disconnect', 'CRITICAL', {
+                'node_name': node_name,
+                'hostname': self._hostname,
+            })
+    
+    def _check_system_shutdown(self, msg: str, syslog_id: str):
+        """Detect system shutdown/reboot."""
+        if 'systemd-journald' in syslog_id or 'systemd' in syslog_id:
+            if 'Journal stopped' in msg or 'Stopping Journal Service' in msg:
+                self._emit('system_shutdown', 'WARNING', {
+                    'reason': 'System journal stopped',
+                    'hostname': self._hostname,
+                })
+            elif 'Shutting down' in msg or 'System is rebooting' in msg:
+                event = 'system_reboot' if 'reboot' in msg.lower() else 'system_shutdown'
+                self._emit(event, 'WARNING', {
+                    'reason': msg[:200],
+                    'hostname': self._hostname,
+                })
+    
+    def _check_permission_change(self, msg: str, syslog_id: str):
+        """Detect user permission changes in PVE."""
+        permission_patterns = [
+            (r'set permissions.*user\s+(\S+)', 'Permission changed'),
+            (r'user added to group.*?(\S+)', 'Added to group'),
+            (r'user removed from group.*?(\S+)', 'Removed from group'),
+            (r'ACL updated.*?(\S+)', 'ACL updated'),
+            (r'Role assigned.*?(\S+)', 'Role assigned'),
+        ]
+        
+        for pattern, action in permission_patterns:
+            match = re.search(pattern, msg, re.IGNORECASE)
+            if match:
+                username = match.group(1)
+                self._emit('user_permission_change', 'INFO', {
+                    'username': username,
+                    'change_details': action,
+                    'hostname': self._hostname,
+                })
+                return
+    
+    def _check_firewall(self, msg: str, syslog_id: str):
+        """Detect firewall issues (not individual drops, but rule errors)."""
+        if re.search(r'pve-firewall.*(?:error|failed|unable)', msg, re.IGNORECASE):
+            self._emit('firewall_issue', 'WARNING', {
+                'reason': msg[:200],
+                'hostname': self._hostname,
+            })
+    
+    # ── Emit helper ──
+    
+    def _emit(self, event_type: str, severity: str, data: Dict):
+        """Emit event to queue with deduplication."""
+        dedup_key = f"{event_type}:{data.get('source_ip', '')}:{data.get('device', '')}:{data.get('service_name', '')}"
+        
+        now = time.time()
+        last = self._recent_events.get(dedup_key, 0)
+        if now - last < self._dedup_window:
+            return  # Skip duplicate
+        
+        self._recent_events[dedup_key] = now
+        
+        # Cleanup old dedup entries periodically
+        if len(self._recent_events) > 200:
+            cutoff = now - self._dedup_window * 2
+            self._recent_events = {
+                k: v for k, v in self._recent_events.items() if v > cutoff
+            }
+        
+        self._queue.put(NotificationEvent(event_type, severity, data, source='journal'))
+
+
+# ─── Task Watcher (Real-time) ────────────────────────────────────
+
+class TaskWatcher:
+    """Watches /var/log/pve/tasks/index for VM/CT and backup events.
+    
+    The PVE task index file is appended when tasks start/finish.
+    Format: UPID:node:pid:pstart:starttime:type:id:user:
+    Final status is recorded when task completes.
+    """
+    
+    TASK_LOG = '/var/log/pve/tasks/index'
+    
+    # Map PVE task types to our event types
+    TASK_MAP = {
+        'qmstart':    ('vm_start',    'INFO'),
+        'qmstop':     ('vm_stop',     'INFO'),
+        'qmshutdown': ('vm_shutdown', 'INFO'),
+        'qmreboot':   ('vm_restart',  'INFO'),
+        'qmreset':    ('vm_restart',  'INFO'),
+        'vzstart':    ('ct_start',    'INFO'),
+        'vzstop':     ('ct_stop',     'INFO'),
+        'vzshutdown': ('ct_stop',     'INFO'),
+        'vzdump':     ('backup_start', 'INFO'),
+        'qmsnapshot': ('snapshot_complete', 'INFO'),
+        'vzsnapshot': ('snapshot_complete', 'INFO'),
+        'qmigrate':   ('migration_start', 'INFO'),
+        'vzmigrate':  ('migration_start', 'INFO'),
+    }
+    
+    def __init__(self, event_queue: Queue):
+        self._queue = event_queue
+        self._running = False
+        self._thread: Optional[threading.Thread] = None
+        self._hostname = _hostname()
+        self._last_position = 0
+    
+    def start(self):
+        if self._running:
+            return
+        self._running = True
+        
+        # Start at end of file
+        if os.path.exists(self.TASK_LOG):
+            try:
+                self._last_position = os.path.getsize(self.TASK_LOG)
+            except OSError:
+                self._last_position = 0
+        
+        self._thread = threading.Thread(target=self._watch_loop, daemon=True,
+                                        name='task-watcher')
+        self._thread.start()
+    
+    def stop(self):
+        self._running = False
+    
+    def _watch_loop(self):
+        """Poll the task index file for new entries."""
+        while self._running:
+            try:
+                if os.path.exists(self.TASK_LOG):
+                    current_size = os.path.getsize(self.TASK_LOG)
+                    
+                    if current_size < self._last_position:
+                        # File was truncated/rotated
+                        self._last_position = 0
+                    
+                    if current_size > self._last_position:
+                        with open(self.TASK_LOG, 'r') as f:
+                            f.seek(self._last_position)
+                            new_lines = f.readlines()
+                            self._last_position = f.tell()
+                        
+                        for line in new_lines:
+                            self._process_task_line(line.strip())
+            except Exception as e:
+                print(f"[TaskWatcher] Error reading task log: {e}")
+            
+            time.sleep(2)  # Check every 2 seconds
+    
+    def _process_task_line(self, line: str):
+        """Process a single task index line.
+        
+        PVE task index format (space-separated):
+        UPID endtime status
+        Where UPID = UPID:node:pid:pstart:starttime:type:id:user:
+        """
+        if not line:
+            return
+        
+        parts = line.split()
+        if not parts:
+            return
+        
+        upid = parts[0]
+        status = parts[2] if len(parts) >= 3 else ''
+        
+        # Parse UPID
+        upid_parts = upid.split(':')
+        if len(upid_parts) < 8:
+            return
+        
+        task_type = upid_parts[5]
+        vmid = upid_parts[6]
+        user = upid_parts[7]
+        
+        # Get VM/CT name
+        vmname = self._get_vm_name(vmid) if vmid else ''
+        
+        # Map to event type
+        event_info = self.TASK_MAP.get(task_type)
+        if not event_info:
+            return
+        
+        event_type, default_severity = event_info
+        
+        # Check if task failed
+        is_error = status and status != 'OK' and status != ''
+        
+        if is_error:
+            # Override to failure event
+            if 'start' in event_type:
+                event_type = event_type.replace('_start', '_fail')
+            elif 'complete' in event_type:
+                event_type = event_type.replace('_complete', '_fail')
+            severity = 'CRITICAL'
+        elif status == 'OK':
+            # Task completed successfully
+            if event_type == 'backup_start':
+                event_type = 'backup_complete'
+            elif event_type == 'migration_start':
+                event_type = 'migration_complete'
+            severity = 'INFO'
+        else:
+            # Task just started (no status yet)
+            severity = default_severity
+        
+        data = {
+            'vmid': vmid,
+            'vmname': vmname or f'ID {vmid}',
+            'hostname': self._hostname,
+            'user': user,
+            'reason': status if is_error else '',
+            'target_node': '',
+            'size': '',
+            'snapshot_name': '',
+        }
+        
+        self._queue.put(NotificationEvent(event_type, severity, data, source='task'))
+    
+    def _get_vm_name(self, vmid: str) -> str:
+        """Try to resolve VMID to name via config files."""
+        if not vmid:
+            return ''
+        
+        # Try QEMU
+        conf_path = f'/etc/pve/qemu-server/{vmid}.conf'
+        name = self._read_name_from_conf(conf_path)
+        if name:
+            return name
+        
+        # Try LXC
+        conf_path = f'/etc/pve/lxc/{vmid}.conf'
+        name = self._read_name_from_conf(conf_path)
+        if name:
+            return name
+        
+        return ''
+    
+    @staticmethod
+    def _read_name_from_conf(path: str) -> str:
+        """Read 'name:' or 'hostname:' from PVE config file."""
+        try:
+            if not os.path.exists(path):
+                return ''
+            with open(path, 'r') as f:
+                for line in f:
+                    if line.startswith('name:'):
+                        return line.split(':', 1)[1].strip()
+                    if line.startswith('hostname:'):
+                        return line.split(':', 1)[1].strip()
+        except (IOError, PermissionError):
+            pass
+        return ''
+
+
+# ─── Polling Collector ────────────────────────────────────────────
+
+class PollingCollector:
+    """Periodic collector that reads Health Monitor pending notifications.
+    
+    Polls health_persistence for:
+    - Pending notification events (state changes from Bloque A)
+    - Unnotified errors
+    - Update availability (every 24h)
+    """
+    
+    def __init__(self, event_queue: Queue, poll_interval: int = 30):
+        self._queue = event_queue
+        self._running = False
+        self._thread: Optional[threading.Thread] = None
+        self._poll_interval = poll_interval
+        self._hostname = _hostname()
+        self._last_update_check = 0
+        self._update_check_interval = 86400  # 24 hours
+    
+    def start(self):
+        if self._running:
+            return
+        self._running = True
+        self._thread = threading.Thread(target=self._poll_loop, daemon=True,
+                                        name='polling-collector')
+        self._thread.start()
+    
+    def stop(self):
+        self._running = False
+    
+    def _poll_loop(self):
+        """Main polling loop."""
+        # Initial delay to let health monitor warm up
+        for _ in range(10):
+            if not self._running:
+                return
+            time.sleep(1)
+        
+        while self._running:
+            try:
+                self._collect_health_events()
+                self._check_updates()
+            except Exception as e:
+                print(f"[PollingCollector] Error: {e}")
+            
+            # Sleep in small increments for responsive shutdown
+            for _ in range(self._poll_interval):
+                if not self._running:
+                    return
+                time.sleep(1)
+    
+    def _collect_health_events(self):
+        """Collect pending notification events from health_persistence."""
+        try:
+            from health_persistence import health_persistence
+            
+            # Get pending notification events
+            events = health_persistence.get_pending_notifications()
+            for evt in events:
+                data = json.loads(evt.get('data', '{}')) if isinstance(evt.get('data'), str) else evt.get('data', {})
+                
+                event_type = evt.get('event_type', 'state_change')
+                severity = data.get('severity', 'WARNING')
+                
+                data['hostname'] = self._hostname
+                data['error_key'] = evt.get('error_key', '')
+                
+                self._queue.put(NotificationEvent(
+                    event_type, severity, data, source='health_monitor'
+                ))
+            
+            # Mark events as notified
+            if events:
+                event_ids = [e['id'] for e in events if 'id' in e]
+                if event_ids:
+                    health_persistence.mark_events_notified(event_ids)
+            
+            # Also check unnotified errors
+            unnotified = health_persistence.get_unnotified_errors()
+            for error in unnotified:
+                self._queue.put(NotificationEvent(
+                    'new_error', error.get('severity', 'WARNING'), {
+                        'category': error.get('category', ''),
+                        'reason': error.get('reason', ''),
+                        'hostname': self._hostname,
+                        'error_key': error.get('error_key', ''),
+                    },
+                    source='health_monitor'
+                ))
+                # Mark as notified
+                if 'id' in error:
+                    health_persistence.mark_notified(error['id'])
+                    
+        except ImportError:
+            pass  # health_persistence not available (CLI mode)
+        except Exception as e:
+            print(f"[PollingCollector] Health event collection error: {e}")
+    
+    def _check_updates(self):
+        """Check for available system updates (every 24h)."""
+        now = time.time()
+        if now - self._last_update_check < self._update_check_interval:
+            return
+        
+        self._last_update_check = now
+        
+        try:
+            result = subprocess.run(
+                ['apt-get', '-s', 'upgrade'],
+                capture_output=True, text=True, timeout=60
+            )
+            
+            if result.returncode == 0:
+                # Count upgradeable packages
+                lines = [l for l in result.stdout.split('\n')
+                         if l.startswith('Inst ')]
+                count = len(lines)
+                
+                if count > 0:
+                    # Show first 5 package names
+                    packages = [l.split()[1] for l in lines[:5]]
+                    details = ', '.join(packages)
+                    if count > 5:
+                        details += f', ... and {count - 5} more'
+                    
+                    self._queue.put(NotificationEvent(
+                        'update_available', 'INFO', {
+                            'count': str(count),
+                            'details': details,
+                            'hostname': self._hostname,
+                        },
+                        source='polling'
+                    ))
+        except Exception:
+            pass  # Non-critical, silently skip

AppImage/scripts/notification_manager.py

@@ -0,0 +1,770 @@
+"""
+ProxMenux Notification Manager
+Central orchestrator for the notification service.
+
+Connects:
+- notification_channels.py  (transport: Telegram, Gotify, Discord)
+- notification_templates.py (message formatting + optional AI)
+- notification_events.py    (event detection: Journal, Task, Polling watchers)
+- health_persistence.py     (DB: config storage, notification_history)
+
+Two interfaces consume this module:
+1. Server mode: Flask imports and calls start()/stop()/send_notification()
+2. CLI mode:    `python3 notification_manager.py --action send --type vm_fail ...`
+                Scripts .sh in /usr/local/share/proxmenux/scripts call this directly.
+
+Author: MacRimi
+"""
+
+import json
+import os
+import sys
+import time
+import socket
+import sqlite3
+import threading
+from queue import Queue, Empty
+from datetime import datetime
+from typing import Dict, Any, List, Optional
+from pathlib import Path
+
+# Ensure local imports work
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+if BASE_DIR not in sys.path:
+    sys.path.insert(0, BASE_DIR)
+
+from notification_channels import create_channel, CHANNEL_TYPES
+from notification_templates import (
+    render_template, format_with_ai, TEMPLATES,
+    EVENT_GROUPS, get_event_types_by_group, get_default_enabled_events
+)
+from notification_events import (
+    JournalWatcher, TaskWatcher, PollingCollector, NotificationEvent
+)
+
+
+# ─── Constants ────────────────────────────────────────────────────
+
+DB_PATH = Path('/usr/local/share/proxmenux/health_monitor.db')
+SETTINGS_PREFIX = 'notification.'
+
+# Cooldown defaults (seconds)
+DEFAULT_COOLDOWNS = {
+    'CRITICAL': 0,       # No cooldown for critical
+    'WARNING':  300,     # 5 min
+    'INFO':     900,     # 15 min
+    'resources': 900,    # 15 min for resource alerts
+    'updates':  86400,   # 24h for update notifications
+}
+
+
+# ─── Notification Manager ─────────────────────────────────────────
+
+class NotificationManager:
+    """Central notification orchestrator.
+    
+    Manages channels, event watchers, deduplication, and dispatch.
+    Can run in server mode (background threads) or CLI mode (one-shot).
+    """
+    
+    def __init__(self):
+        self._channels: Dict[str, Any] = {}  # channel_name -> channel_instance
+        self._event_queue: Queue = Queue()
+        self._running = False
+        self._config: Dict[str, str] = {}
+        self._enabled = False
+        self._lock = threading.Lock()
+        
+        # Watchers
+        self._journal_watcher: Optional[JournalWatcher] = None
+        self._task_watcher: Optional[TaskWatcher] = None
+        self._polling_collector: Optional[PollingCollector] = None
+        self._dispatch_thread: Optional[threading.Thread] = None
+        
+        # Cooldown tracking: {event_type_or_key: last_sent_timestamp}
+        self._cooldowns: Dict[str, float] = {}
+        
+        # Stats
+        self._stats = {
+            'started_at': None,
+            'total_sent': 0,
+            'total_errors': 0,
+            'last_sent_at': None,
+        }
+    
+    # ─── Configuration ──────────────────────────────────────────
+    
+    def _load_config(self):
+        """Load notification settings from the shared SQLite database."""
+        self._config = {}
+        try:
+            if not DB_PATH.exists():
+                return
+            
+            conn = sqlite3.connect(str(DB_PATH), timeout=10)
+            conn.execute('PRAGMA journal_mode=WAL')
+            conn.execute('PRAGMA busy_timeout=5000')
+            cursor = conn.cursor()
+            cursor.execute(
+                'SELECT setting_key, setting_value FROM user_settings WHERE setting_key LIKE ?',
+                (f'{SETTINGS_PREFIX}%',)
+            )
+            for key, value in cursor.fetchall():
+                # Strip prefix for internal use
+                short_key = key[len(SETTINGS_PREFIX):]
+                self._config[short_key] = value
+            conn.close()
+        except Exception as e:
+            print(f"[NotificationManager] Failed to load config: {e}")
+        
+        self._enabled = self._config.get('enabled', 'false') == 'true'
+        self._rebuild_channels()
+    
+    def _save_setting(self, key: str, value: str):
+        """Save a single notification setting to the database."""
+        full_key = f'{SETTINGS_PREFIX}{key}'
+        now = datetime.now().isoformat()
+        try:
+            conn = sqlite3.connect(str(DB_PATH), timeout=10)
+            conn.execute('PRAGMA journal_mode=WAL')
+            conn.execute('PRAGMA busy_timeout=5000')
+            cursor = conn.cursor()
+            cursor.execute('''
+                INSERT OR REPLACE INTO user_settings (setting_key, setting_value, updated_at)
+                VALUES (?, ?, ?)
+            ''', (full_key, value, now))
+            conn.commit()
+            conn.close()
+            self._config[key] = value
+        except Exception as e:
+            print(f"[NotificationManager] Failed to save setting {key}: {e}")
+    
+    def _rebuild_channels(self):
+        """Rebuild channel instances from current config."""
+        self._channels = {}
+        
+        for ch_type in CHANNEL_TYPES:
+            enabled_key = f'{ch_type}.enabled'
+            if self._config.get(enabled_key) != 'true':
+                continue
+            
+            # Gather config keys for this channel
+            ch_config = {}
+            for config_key in CHANNEL_TYPES[ch_type]['config_keys']:
+                full_key = f'{ch_type}.{config_key}'
+                ch_config[config_key] = self._config.get(full_key, '')
+            
+            channel = create_channel(ch_type, ch_config)
+            if channel:
+                valid, err = channel.validate_config()
+                if valid:
+                    self._channels[ch_type] = channel
+                else:
+                    print(f"[NotificationManager] Channel {ch_type} invalid: {err}")
+    
+    def reload_config(self):
+        """Reload config from DB without restarting."""
+        with self._lock:
+            self._load_config()
+        return {'success': True, 'channels': list(self._channels.keys())}
+    
+    # ─── Server Mode (Background) ──────────────────────────────
+    
+    def start(self):
+        """Start the notification service in server mode.
+        
+        Launches watchers and dispatch loop as daemon threads.
+        Called by flask_server.py on startup.
+        """
+        if self._running:
+            return
+        
+        self._load_config()
+        
+        if not self._enabled:
+            print("[NotificationManager] Service is disabled. Skipping start.")
+            return
+        
+        self._running = True
+        self._stats['started_at'] = datetime.now().isoformat()
+        
+        # Start event watchers
+        self._journal_watcher = JournalWatcher(self._event_queue)
+        self._task_watcher = TaskWatcher(self._event_queue)
+        self._polling_collector = PollingCollector(self._event_queue)
+        
+        self._journal_watcher.start()
+        self._task_watcher.start()
+        self._polling_collector.start()
+        
+        # Start dispatch loop
+        self._dispatch_thread = threading.Thread(
+            target=self._dispatch_loop, daemon=True, name='notification-dispatch'
+        )
+        self._dispatch_thread.start()
+        
+        print(f"[NotificationManager] Started with channels: {list(self._channels.keys())}")
+    
+    def stop(self):
+        """Stop the notification service cleanly."""
+        self._running = False
+        
+        if self._journal_watcher:
+            self._journal_watcher.stop()
+        if self._task_watcher:
+            self._task_watcher.stop()
+        if self._polling_collector:
+            self._polling_collector.stop()
+        
+        print("[NotificationManager] Stopped.")
+    
+    def _dispatch_loop(self):
+        """Main dispatch loop: reads queue -> filters -> formats -> sends -> records."""
+        while self._running:
+            try:
+                event = self._event_queue.get(timeout=2)
+            except Empty:
+                continue
+            
+            try:
+                self._process_event(event)
+            except Exception as e:
+                print(f"[NotificationManager] Dispatch error: {e}")
+    
+    def _process_event(self, event: NotificationEvent):
+        """Process a single event from the queue."""
+        if not self._enabled:
+            return
+        
+        # Check if this event type is enabled in settings
+        event_setting = f'events.{event.event_type}'
+        if self._config.get(event_setting, 'true') == 'false':
+            return
+        
+        # Check severity filter
+        min_severity = self._config.get('filter.min_severity', 'INFO')
+        if not self._meets_severity(event.severity, min_severity):
+            return
+        
+        # Check cooldown
+        if not self._check_cooldown(event):
+            return
+        
+        # Render message from template
+        rendered = render_template(event.event_type, event.data)
+        
+        # Optional AI enhancement
+        ai_config = {
+            'enabled': self._config.get('ai_enabled', 'false'),
+            'provider': self._config.get('ai_provider', ''),
+            'api_key': self._config.get('ai_api_key', ''),
+            'model': self._config.get('ai_model', ''),
+        }
+        body = format_with_ai(
+            rendered['title'], rendered['body'], rendered['severity'], ai_config
+        )
+        
+        # Send through all active channels
+        self._dispatch_to_channels(
+            rendered['title'], body, rendered['severity'],
+            event.event_type, event.data, event.source
+        )
+    
+    def _dispatch_to_channels(self, title: str, body: str, severity: str,
+                               event_type: str, data: Dict, source: str):
+        """Send notification through all configured channels."""
+        with self._lock:
+            channels = dict(self._channels)
+        
+        for ch_name, channel in channels.items():
+            try:
+                result = channel.send(title, body, severity, data)
+                self._record_history(
+                    event_type, ch_name, title, body, severity,
+                    result.get('success', False),
+                    result.get('error', ''),
+                    source
+                )
+                
+                if result.get('success'):
+                    self._stats['total_sent'] += 1
+                    self._stats['last_sent_at'] = datetime.now().isoformat()
+                else:
+                    self._stats['total_errors'] += 1
+                    print(f"[NotificationManager] Send failed ({ch_name}): {result.get('error')}")
+                    
+            except Exception as e:
+                self._stats['total_errors'] += 1
+                self._record_history(
+                    event_type, ch_name, title, body, severity,
+                    False, str(e), source
+                )
+    
+    # ─── Cooldown / Dedup ───────────────────────────────────────
+    
+    def _check_cooldown(self, event: NotificationEvent) -> bool:
+        """Check if the event passes cooldown rules."""
+        now = time.time()
+        
+        # Determine cooldown period
+        template = TEMPLATES.get(event.event_type, {})
+        group = template.get('group', 'system')
+        
+        # Priority: per-type config > per-severity > default
+        cooldown_key = f'cooldown.{event.event_type}'
+        cooldown_str = self._config.get(cooldown_key)
+        
+        if cooldown_str is None:
+            cooldown_key_group = f'cooldown.{group}'
+            cooldown_str = self._config.get(cooldown_key_group)
+        
+        if cooldown_str is not None:
+            cooldown = int(cooldown_str)
+        else:
+            cooldown = DEFAULT_COOLDOWNS.get(event.severity, 300)
+        
+        # CRITICAL events have zero cooldown by default
+        if event.severity == 'CRITICAL' and cooldown_str is None:
+            cooldown = 0
+        
+        # Check against last sent time
+        dedup_key = f"{event.event_type}:{event.data.get('category', '')}:{event.data.get('vmid', '')}"
+        last_sent = self._cooldowns.get(dedup_key, 0)
+        
+        if now - last_sent < cooldown:
+            return False
+        
+        self._cooldowns[dedup_key] = now
+        return True
+    
+    @staticmethod
+    def _meets_severity(event_severity: str, min_severity: str) -> bool:
+        """Check if event severity meets the minimum threshold."""
+        levels = {'INFO': 0, 'WARNING': 1, 'CRITICAL': 2}
+        return levels.get(event_severity, 0) >= levels.get(min_severity, 0)
+    
+    # ─── History Recording ──────────────────────────────────────
+    
+    def _record_history(self, event_type: str, channel: str, title: str,
+                        message: str, severity: str, success: bool,
+                        error_message: str, source: str):
+        """Record a notification attempt in the history table."""
+        try:
+            conn = sqlite3.connect(str(DB_PATH), timeout=10)
+            conn.execute('PRAGMA journal_mode=WAL')
+            conn.execute('PRAGMA busy_timeout=5000')
+            cursor = conn.cursor()
+            cursor.execute('''
+                INSERT INTO notification_history
+                (event_type, channel, title, message, severity, sent_at, success, error_message, source)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+            ''', (
+                event_type, channel, title, message[:500], severity,
+                datetime.now().isoformat(), 1 if success else 0,
+                error_message[:500] if error_message else None, source
+            ))
+            conn.commit()
+            conn.close()
+        except Exception as e:
+            print(f"[NotificationManager] History record error: {e}")
+    
+    # ─── Public API (used by Flask routes and CLI) ──────────────
+    
+    def send_notification(self, event_type: str, severity: str,
+                          title: str, message: str,
+                          data: Optional[Dict] = None,
+                          source: str = 'api') -> Dict[str, Any]:
+        """Send a notification directly (bypasses queue and cooldown).
+        
+        Used by CLI and API for explicit sends.
+        """
+        if not self._channels:
+            self._load_config()
+        
+        if not self._channels:
+            return {
+                'success': False,
+                'error': 'No channels configured or enabled',
+                'channels_sent': [],
+            }
+        
+        # Render template if available
+        if event_type in TEMPLATES and not message:
+            rendered = render_template(event_type, data or {})
+            title = title or rendered['title']
+            message = rendered['body']
+            severity = severity or rendered['severity']
+        
+        # AI enhancement
+        ai_config = {
+            'enabled': self._config.get('ai_enabled', 'false'),
+            'provider': self._config.get('ai_provider', ''),
+            'api_key': self._config.get('ai_api_key', ''),
+            'model': self._config.get('ai_model', ''),
+        }
+        message = format_with_ai(title, message, severity, ai_config)
+        
+        results = {}
+        channels_sent = []
+        errors = []
+        
+        with self._lock:
+            channels = dict(self._channels)
+        
+        for ch_name, channel in channels.items():
+            try:
+                result = channel.send(title, message, severity, data)
+                results[ch_name] = result
+                
+                self._record_history(
+                    event_type, ch_name, title, message, severity,
+                    result.get('success', False),
+                    result.get('error', ''),
+                    source
+                )
+                
+                if result.get('success'):
+                    channels_sent.append(ch_name)
+                else:
+                    errors.append(f"{ch_name}: {result.get('error')}")
+            except Exception as e:
+                errors.append(f"{ch_name}: {str(e)}")
+        
+        return {
+            'success': len(channels_sent) > 0,
+            'channels_sent': channels_sent,
+            'errors': errors,
+            'total_channels': len(channels),
+        }
+    
+    def send_raw(self, title: str, message: str,
+                 severity: str = 'INFO',
+                 source: str = 'api') -> Dict[str, Any]:
+        """Send a raw message without template (for custom scripts)."""
+        return self.send_notification(
+            'custom', severity, title, message, source=source
+        )
+    
+    def test_channel(self, channel_name: str = 'all') -> Dict[str, Any]:
+        """Test one or all configured channels."""
+        if not self._channels:
+            self._load_config()
+        
+        if not self._channels:
+            return {'success': False, 'error': 'No channels configured'}
+        
+        results = {}
+        
+        if channel_name == 'all':
+            targets = dict(self._channels)
+        elif channel_name in self._channels:
+            targets = {channel_name: self._channels[channel_name]}
+        else:
+            # Try to create channel from config even if not enabled
+            ch_config = {}
+            for config_key in CHANNEL_TYPES.get(channel_name, {}).get('config_keys', []):
+                ch_config[config_key] = self._config.get(f'{channel_name}.{config_key}', '')
+            
+            channel = create_channel(channel_name, ch_config)
+            if channel:
+                targets = {channel_name: channel}
+            else:
+                return {'success': False, 'error': f'Channel {channel_name} not configured'}
+        
+        for ch_name, channel in targets.items():
+            success, error = channel.test()
+            results[ch_name] = {'success': success, 'error': error}
+            
+            self._record_history(
+                'test', ch_name, 'ProxMenux Test',
+                'Test notification', 'INFO',
+                success, error, 'api'
+            )
+        
+        overall_success = any(r['success'] for r in results.values())
+        return {
+            'success': overall_success,
+            'results': results,
+        }
+    
+    def get_status(self) -> Dict[str, Any]:
+        """Get current service status."""
+        if not self._config:
+            self._load_config()
+        
+        return {
+            'enabled': self._enabled,
+            'running': self._running,
+            'channels': {
+                name: {
+                    'type': name,
+                    'connected': True,
+                }
+                for name in self._channels
+            },
+            'stats': self._stats,
+            'watchers': {
+                'journal': self._journal_watcher is not None and self._running,
+                'task': self._task_watcher is not None and self._running,
+                'polling': self._polling_collector is not None and self._running,
+            },
+        }
+    
+    def set_enabled(self, enabled: bool) -> Dict[str, Any]:
+        """Enable or disable the notification service."""
+        self._save_setting('enabled', 'true' if enabled else 'false')
+        self._enabled = enabled
+        
+        if enabled and not self._running:
+            self.start()
+        elif not enabled and self._running:
+            self.stop()
+        
+        return {'success': True, 'enabled': enabled}
+    
+    def list_channels(self) -> Dict[str, Any]:
+        """List all channel types with their configuration status."""
+        if not self._config:
+            self._load_config()
+        
+        channels_info = {}
+        for ch_type, info in CHANNEL_TYPES.items():
+            enabled = self._config.get(f'{ch_type}.enabled', 'false') == 'true'
+            configured = all(
+                bool(self._config.get(f'{ch_type}.{k}', ''))
+                for k in info['config_keys']
+            )
+            channels_info[ch_type] = {
+                'name': info['name'],
+                'enabled': enabled,
+                'configured': configured,
+                'active': ch_type in self._channels,
+            }
+        
+        return {'channels': channels_info}
+    
+    def get_history(self, limit: int = 50, offset: int = 0,
+                    severity: str = '', channel: str = '') -> Dict[str, Any]:
+        """Get notification history with optional filters."""
+        try:
+            conn = sqlite3.connect(str(DB_PATH), timeout=10)
+            conn.execute('PRAGMA journal_mode=WAL')
+            conn.execute('PRAGMA busy_timeout=5000')
+            conn.row_factory = sqlite3.Row
+            cursor = conn.cursor()
+            
+            query = 'SELECT * FROM notification_history WHERE 1=1'
+            params: list = []
+            
+            if severity:
+                query += ' AND severity = ?'
+                params.append(severity)
+            if channel:
+                query += ' AND channel = ?'
+                params.append(channel)
+            
+            query += ' ORDER BY sent_at DESC LIMIT ? OFFSET ?'
+            params.extend([limit, offset])
+            
+            cursor.execute(query, params)
+            rows = [dict(row) for row in cursor.fetchall()]
+            
+            # Get total count
+            count_query = 'SELECT COUNT(*) FROM notification_history WHERE 1=1'
+            count_params: list = []
+            if severity:
+                count_query += ' AND severity = ?'
+                count_params.append(severity)
+            if channel:
+                count_query += ' AND channel = ?'
+                count_params.append(channel)
+            
+            cursor.execute(count_query, count_params)
+            total = cursor.fetchone()[0]
+            
+            conn.close()
+            
+            return {
+                'history': rows,
+                'total': total,
+                'limit': limit,
+                'offset': offset,
+            }
+        except Exception as e:
+            return {'history': [], 'total': 0, 'error': str(e)}
+    
+    def clear_history(self) -> Dict[str, Any]:
+        """Clear all notification history."""
+        try:
+            conn = sqlite3.connect(str(DB_PATH), timeout=10)
+            conn.execute('PRAGMA journal_mode=WAL')
+            conn.execute('PRAGMA busy_timeout=5000')
+            conn.execute('DELETE FROM notification_history')
+            conn.commit()
+            conn.close()
+            return {'success': True}
+        except Exception as e:
+            return {'success': False, 'error': str(e)}
+    
+    def get_settings(self) -> Dict[str, Any]:
+        """Get all notification settings for the UI."""
+        if not self._config:
+            self._load_config()
+        
+        return {
+            'enabled': self._enabled,
+            'settings': {f'{SETTINGS_PREFIX}{k}': v for k, v in self._config.items()},
+            'channels': self.list_channels()['channels'],
+            'event_groups': EVENT_GROUPS,
+            'event_types': get_event_types_by_group(),
+            'default_events': get_default_enabled_events(),
+        }
+    
+    def save_settings(self, settings: Dict[str, str]) -> Dict[str, Any]:
+        """Save multiple notification settings at once."""
+        try:
+            conn = sqlite3.connect(str(DB_PATH), timeout=10)
+            conn.execute('PRAGMA journal_mode=WAL')
+            conn.execute('PRAGMA busy_timeout=5000')
+            cursor = conn.cursor()
+            now = datetime.now().isoformat()
+            
+            for key, value in settings.items():
+                # Accept both prefixed and unprefixed keys
+                full_key = key if key.startswith(SETTINGS_PREFIX) else f'{SETTINGS_PREFIX}{key}'
+                short_key = full_key[len(SETTINGS_PREFIX):]
+                
+                cursor.execute('''
+                    INSERT OR REPLACE INTO user_settings (setting_key, setting_value, updated_at)
+                    VALUES (?, ?, ?)
+                ''', (full_key, str(value), now))
+                
+                self._config[short_key] = str(value)
+            
+            conn.commit()
+            conn.close()
+            
+            # Rebuild channels with new config
+            self._enabled = self._config.get('enabled', 'false') == 'true'
+            self._rebuild_channels()
+            
+            return {'success': True, 'channels_active': list(self._channels.keys())}
+        except Exception as e:
+            return {'success': False, 'error': str(e)}
+
+
+# ─── Singleton (for server mode) ─────────────────────────────────
+
+notification_manager = NotificationManager()
+
+
+# ─── CLI Interface ────────────────────────────────────────────────
+
+def _print_result(result: Dict, as_json: bool):
+    """Print CLI result in human-readable or JSON format."""
+    if as_json:
+        print(json.dumps(result, indent=2, default=str))
+        return
+    
+    if result.get('success'):
+        print(f"OK: ", end='')
+    elif 'success' in result and not result['success']:
+        print(f"ERROR: ", end='')
+    
+    # Format based on content
+    if 'channels_sent' in result:
+        sent = result.get('channels_sent', [])
+        print(f"Sent via: {', '.join(sent) if sent else 'none'}")
+        if result.get('errors'):
+            for err in result['errors']:
+                print(f"  Error: {err}")
+    elif 'results' in result:
+        for ch, r in result['results'].items():
+            status = 'OK' if r['success'] else f"FAILED: {r['error']}"
+            print(f"  {ch}: {status}")
+    elif 'channels' in result:
+        for ch, info in result['channels'].items():
+            status = 'active' if info.get('active') else ('configured' if info.get('configured') else 'not configured')
+            enabled = 'enabled' if info.get('enabled') else 'disabled'
+            print(f"  {info['name']}: {enabled}, {status}")
+    elif 'enabled' in result and 'running' in result:
+        print(f"Enabled: {result['enabled']}, Running: {result['running']}")
+        if result.get('stats'):
+            stats = result['stats']
+            print(f"  Total sent: {stats.get('total_sent', 0)}")
+            print(f"  Total errors: {stats.get('total_errors', 0)}")
+            if stats.get('last_sent_at'):
+                print(f"  Last sent: {stats['last_sent_at']}")
+    elif 'enabled' in result:
+        print(f"Service {'enabled' if result['enabled'] else 'disabled'}")
+    else:
+        print(json.dumps(result, indent=2, default=str))
+
+
+if __name__ == '__main__':
+    import argparse
+    
+    parser = argparse.ArgumentParser(
+        description='ProxMenux Notification Manager CLI',
+        epilog='Example: python3 notification_manager.py --action send --type vm_fail --severity CRITICAL --title "VM 100 failed" --message "QEMU process crashed"'
+    )
+    parser.add_argument('--action', required=True,
+                        choices=['send', 'send-raw', 'test', 'status',
+                                 'enable', 'disable', 'list-channels'],
+                        help='Action to perform')
+    parser.add_argument('--type', help='Event type for send action (e.g. vm_fail, backup_complete)')
+    parser.add_argument('--severity', default='INFO',
+                        choices=['INFO', 'WARNING', 'CRITICAL'],
+                        help='Notification severity (default: INFO)')
+    parser.add_argument('--title', help='Notification title')
+    parser.add_argument('--message', help='Notification message body')
+    parser.add_argument('--channel', default='all',
+                        help='Specific channel for test (default: all)')
+    parser.add_argument('--json', action='store_true',
+                        help='Output result as JSON')
+    
+    args = parser.parse_args()
+    
+    mgr = NotificationManager()
+    mgr._load_config()
+    
+    if args.action == 'send':
+        if not args.type:
+            parser.error('--type is required for send action')
+        result = mgr.send_notification(
+            args.type, args.severity,
+            args.title or '', args.message or '',
+            data={
+                'hostname': socket.gethostname().split('.')[0],
+                'reason': args.message or '',
+            },
+            source='cli'
+        )
+    
+    elif args.action == 'send-raw':
+        if not args.title or not args.message:
+            parser.error('--title and --message are required for send-raw')
+        result = mgr.send_raw(args.title, args.message, args.severity, source='cli')
+    
+    elif args.action == 'test':
+        result = mgr.test_channel(args.channel)
+    
+    elif args.action == 'status':
+        result = mgr.get_status()
+    
+    elif args.action == 'enable':
+        result = mgr.set_enabled(True)
+    
+    elif args.action == 'disable':
+        result = mgr.set_enabled(False)
+    
+    elif args.action == 'list-channels':
+        result = mgr.list_channels()
+    
+    else:
+        result = {'error': f'Unknown action: {args.action}'}
+    
+    _print_result(result, args.json)
+    
+    # Exit with appropriate code
+    sys.exit(0 if result.get('success', True) else 1)

AppImage/scripts/notification_templates.py

@@ -0,0 +1,554 @@
+"""
+ProxMenux Notification Templates
+Message templates for all event types with per-channel formatting.
+
+Templates use Python str.format() variables:
+  {hostname}, {severity}, {category}, {reason}, {summary},
+  {previous}, {current}, {vmid}, {vmname}, {timestamp}, etc.
+
+Optional AI enhancement enriches messages with context/suggestions.
+
+Author: MacRimi
+"""
+
+import json
+import socket
+import time
+import urllib.request
+import urllib.error
+from typing import Dict, Any, Optional
+
+
+# ─── Severity Icons ──────────────────────────────────────────────
+
+SEVERITY_ICONS = {
+    'CRITICAL': '\U0001F534',
+    'WARNING':  '\U0001F7E1',
+    'INFO':     '\U0001F535',
+    'OK':       '\U0001F7E2',
+    'UNKNOWN':  '\u26AA',
+}
+
+SEVERITY_ICONS_DISCORD = {
+    'CRITICAL': ':red_circle:',
+    'WARNING':  ':yellow_circle:',
+    'INFO':     ':blue_circle:',
+    'OK':       ':green_circle:',
+    'UNKNOWN':  ':white_circle:',
+}
+
+
+# ─── Event Templates ─────────────────────────────────────────────
+# Each template has a 'title' and 'body' with {variable} placeholders.
+# 'group' is used for UI event filter grouping.
+# 'default_enabled' controls initial state in settings.
+
+TEMPLATES = {
+    # ── Health Monitor state changes ──
+    'state_change': {
+        'title': '{hostname}: {category} changed to {current}',
+        'body': '{category} status changed from {previous} to {current}.\n{reason}',
+        'group': 'system',
+        'default_enabled': True,
+    },
+    'new_error': {
+        'title': '{hostname}: New {severity} - {category}',
+        'body': '{reason}',
+        'group': 'system',
+        'default_enabled': True,
+    },
+    'error_resolved': {
+        'title': '{hostname}: Resolved - {category}',
+        'body': '{reason}\nDuration: {duration}',
+        'group': 'system',
+        'default_enabled': True,
+    },
+    'error_escalated': {
+        'title': '{hostname}: Escalated to {severity} - {category}',
+        'body': '{reason}',
+        'group': 'system',
+        'default_enabled': True,
+    },
+    
+    # ── VM / CT events ──
+    'vm_start': {
+        'title': '{hostname}: VM {vmid} started',
+        'body': '{vmname} ({vmid}) has been started.',
+        'group': 'vm_ct',
+        'default_enabled': True,
+    },
+    'vm_stop': {
+        'title': '{hostname}: VM {vmid} stopped',
+        'body': '{vmname} ({vmid}) has been stopped.',
+        'group': 'vm_ct',
+        'default_enabled': False,
+    },
+    'vm_shutdown': {
+        'title': '{hostname}: VM {vmid} shutdown',
+        'body': '{vmname} ({vmid}) has been shut down.',
+        'group': 'vm_ct',
+        'default_enabled': False,
+    },
+    'vm_fail': {
+        'title': '{hostname}: VM {vmid} FAILED',
+        'body': '{vmname} ({vmid}) has failed.\n{reason}',
+        'group': 'vm_ct',
+        'default_enabled': True,
+    },
+    'vm_restart': {
+        'title': '{hostname}: VM {vmid} restarted',
+        'body': '{vmname} ({vmid}) has been restarted.',
+        'group': 'vm_ct',
+        'default_enabled': False,
+    },
+    'ct_start': {
+        'title': '{hostname}: CT {vmid} started',
+        'body': '{vmname} ({vmid}) has been started.',
+        'group': 'vm_ct',
+        'default_enabled': True,
+    },
+    'ct_stop': {
+        'title': '{hostname}: CT {vmid} stopped',
+        'body': '{vmname} ({vmid}) has been stopped.',
+        'group': 'vm_ct',
+        'default_enabled': False,
+    },
+    'ct_fail': {
+        'title': '{hostname}: CT {vmid} FAILED',
+        'body': '{vmname} ({vmid}) has failed.\n{reason}',
+        'group': 'vm_ct',
+        'default_enabled': True,
+    },
+    'migration_start': {
+        'title': '{hostname}: Migration started - {vmid}',
+        'body': '{vmname} ({vmid}) migration to {target_node} started.',
+        'group': 'vm_ct',
+        'default_enabled': True,
+    },
+    'migration_complete': {
+        'title': '{hostname}: Migration complete - {vmid}',
+        'body': '{vmname} ({vmid}) migrated successfully to {target_node}.',
+        'group': 'vm_ct',
+        'default_enabled': True,
+    },
+    'migration_fail': {
+        'title': '{hostname}: Migration FAILED - {vmid}',
+        'body': '{vmname} ({vmid}) migration to {target_node} failed.\n{reason}',
+        'group': 'vm_ct',
+        'default_enabled': True,
+    },
+    
+    # ── Backup / Snapshot events ──
+    'backup_start': {
+        'title': '{hostname}: Backup started - {vmid}',
+        'body': 'Backup of {vmname} ({vmid}) has started.',
+        'group': 'backup',
+        'default_enabled': False,
+    },
+    'backup_complete': {
+        'title': '{hostname}: Backup complete - {vmid}',
+        'body': 'Backup of {vmname} ({vmid}) completed successfully.\nSize: {size}',
+        'group': 'backup',
+        'default_enabled': True,
+    },
+    'backup_fail': {
+        'title': '{hostname}: Backup FAILED - {vmid}',
+        'body': 'Backup of {vmname} ({vmid}) has failed.\n{reason}',
+        'group': 'backup',
+        'default_enabled': True,
+    },
+    'snapshot_complete': {
+        'title': '{hostname}: Snapshot created - {vmid}',
+        'body': 'Snapshot of {vmname} ({vmid}) created: {snapshot_name}',
+        'group': 'backup',
+        'default_enabled': False,
+    },
+    'snapshot_fail': {
+        'title': '{hostname}: Snapshot FAILED - {vmid}',
+        'body': 'Snapshot of {vmname} ({vmid}) failed.\n{reason}',
+        'group': 'backup',
+        'default_enabled': True,
+    },
+    
+    # ── Resource events (from Health Monitor) ──
+    'cpu_high': {
+        'title': '{hostname}: High CPU usage ({value}%)',
+        'body': 'CPU usage is at {value}% on {cores} cores.\n{details}',
+        'group': 'resources',
+        'default_enabled': True,
+    },
+    'ram_high': {
+        'title': '{hostname}: High memory usage ({value}%)',
+        'body': 'Memory usage: {used} / {total} ({value}%).\n{details}',
+        'group': 'resources',
+        'default_enabled': True,
+    },
+    'temp_high': {
+        'title': '{hostname}: High temperature ({value}C)',
+        'body': 'CPU temperature: {value}C (threshold: {threshold}C).\n{details}',
+        'group': 'resources',
+        'default_enabled': True,
+    },
+    'disk_space_low': {
+        'title': '{hostname}: Low disk space on {mount}',
+        'body': '{mount}: {used}% used ({available} available).',
+        'group': 'storage',
+        'default_enabled': True,
+    },
+    'disk_io_error': {
+        'title': '{hostname}: Disk I/O error',
+        'body': 'I/O error detected on {device}.\n{reason}',
+        'group': 'storage',
+        'default_enabled': True,
+    },
+    'load_high': {
+        'title': '{hostname}: High system load ({value})',
+        'body': 'System load average: {value} on {cores} cores.\n{details}',
+        'group': 'resources',
+        'default_enabled': True,
+    },
+    
+    # ── Network events ──
+    'network_down': {
+        'title': '{hostname}: Network connectivity lost',
+        'body': 'Network connectivity check failed.\n{reason}',
+        'group': 'network',
+        'default_enabled': True,
+    },
+    'network_latency': {
+        'title': '{hostname}: High network latency ({value}ms)',
+        'body': 'Latency to gateway: {value}ms (threshold: {threshold}ms).',
+        'group': 'network',
+        'default_enabled': False,
+    },
+    
+    # ── Security events ──
+    'auth_fail': {
+        'title': '{hostname}: Authentication failure',
+        'body': 'Failed login attempt from {source_ip}.\nUser: {username}\nService: {service}',
+        'group': 'security',
+        'default_enabled': True,
+    },
+    'ip_block': {
+        'title': '{hostname}: IP blocked by Fail2Ban',
+        'body': 'IP {source_ip} has been banned.\nJail: {jail}\nFailures: {failures}',
+        'group': 'security',
+        'default_enabled': True,
+    },
+    'firewall_issue': {
+        'title': '{hostname}: Firewall issue detected',
+        'body': '{reason}',
+        'group': 'security',
+        'default_enabled': True,
+    },
+    'user_permission_change': {
+        'title': '{hostname}: User permission changed',
+        'body': 'User: {username}\nChange: {change_details}',
+        'group': 'security',
+        'default_enabled': True,
+    },
+    
+    # ── Cluster events ──
+    'split_brain': {
+        'title': '{hostname}: SPLIT-BRAIN detected',
+        'body': 'Cluster split-brain condition detected.\nQuorum status: {quorum}',
+        'group': 'cluster',
+        'default_enabled': True,
+    },
+    'node_disconnect': {
+        'title': '{hostname}: Node disconnected',
+        'body': 'Node {node_name} has disconnected from the cluster.',
+        'group': 'cluster',
+        'default_enabled': True,
+    },
+    'node_reconnect': {
+        'title': '{hostname}: Node reconnected',
+        'body': 'Node {node_name} has reconnected to the cluster.',
+        'group': 'cluster',
+        'default_enabled': True,
+    },
+    
+    # ── System events ──
+    'system_shutdown': {
+        'title': '{hostname}: System shutting down',
+        'body': 'The system is shutting down.\n{reason}',
+        'group': 'system',
+        'default_enabled': True,
+    },
+    'system_reboot': {
+        'title': '{hostname}: System rebooting',
+        'body': 'The system is rebooting.\n{reason}',
+        'group': 'system',
+        'default_enabled': True,
+    },
+    'system_problem': {
+        'title': '{hostname}: System problem detected',
+        'body': '{reason}',
+        'group': 'system',
+        'default_enabled': True,
+    },
+    'service_fail': {
+        'title': '{hostname}: Service failed - {service_name}',
+        'body': 'Service {service_name} has failed.\n{reason}',
+        'group': 'system',
+        'default_enabled': True,
+    },
+    'update_available': {
+        'title': '{hostname}: Updates available ({count})',
+        'body': '{count} package updates are available.\n{details}',
+        'group': 'system',
+        'default_enabled': False,
+    },
+    'update_complete': {
+        'title': '{hostname}: Update completed',
+        'body': '{details}',
+        'group': 'system',
+        'default_enabled': False,
+    },
+    
+    # ── Unknown persistent (from health monitor) ──
+    'unknown_persistent': {
+        'title': '{hostname}: Check unavailable - {category}',
+        'body': 'Health check for {category} has been unavailable for 3+ cycles.\n{reason}',
+        'group': 'system',
+        'default_enabled': False,
+    },
+}
+
+# ─── Event Groups (for UI filtering) ─────────────────────────────
+
+EVENT_GROUPS = {
+    'system':    {'label': 'System',     'description': 'System health, services, updates'},
+    'vm_ct':     {'label': 'VM / CT',    'description': 'Virtual machines and containers'},
+    'backup':    {'label': 'Backup',     'description': 'Backups and snapshots'},
+    'resources': {'label': 'Resources',  'description': 'CPU, memory, temperature, load'},
+    'storage':   {'label': 'Storage',    'description': 'Disk space and I/O'},
+    'network':   {'label': 'Network',    'description': 'Connectivity and latency'},
+    'security':  {'label': 'Security',   'description': 'Authentication, firewall, bans'},
+    'cluster':   {'label': 'Cluster',    'description': 'Cluster health and quorum'},
+}
+
+
+# ─── Template Renderer ───────────────────────────────────────────
+
+def _get_hostname() -> str:
+    """Get short hostname for message titles."""
+    try:
+        return socket.gethostname().split('.')[0]
+    except Exception:
+        return 'proxmox'
+
+
+def render_template(event_type: str, data: Dict[str, Any]) -> Dict[str, str]:
+    """Render a template with the given data.
+    
+    Args:
+        event_type: Key from TEMPLATES dict
+        data: Variables to fill into the template
+    
+    Returns:
+        {'title': rendered_title, 'body': rendered_body, 'severity': severity}
+    """
+    template = TEMPLATES.get(event_type)
+    if not template:
+        # Fallback for unknown event types
+        return {
+            'title': f"{_get_hostname()}: {event_type}",
+            'body': data.get('message', data.get('reason', str(data))),
+            'severity': data.get('severity', 'INFO'),
+        }
+    
+    # Ensure hostname is always available
+    variables = {
+        'hostname': _get_hostname(),
+        'timestamp': time.strftime('%Y-%m-%d %H:%M:%S'),
+        'severity': data.get('severity', 'INFO'),
+        # Common defaults
+        'vmid': '',
+        'vmname': '',
+        'reason': '',
+        'summary': '',
+        'details': '',
+        'category': '',
+        'previous': '',
+        'current': '',
+        'duration': '',
+        'value': '',
+        'threshold': '',
+        'source_ip': '',
+        'username': '',
+        'service': '',
+        'service_name': '',
+        'node_name': '',
+        'target_node': '',
+        'mount': '',
+        'device': '',
+        'used': '',
+        'total': '',
+        'available': '',
+        'cores': '',
+        'count': '',
+        'size': '',
+        'snapshot_name': '',
+        'jail': '',
+        'failures': '',
+        'quorum': '',
+        'change_details': '',
+        'message': '',
+    }
+    variables.update(data)
+    
+    try:
+        title = template['title'].format(**variables)
+    except (KeyError, ValueError):
+        title = template['title']  # Use raw template if formatting fails
+    
+    try:
+        body = template['body'].format(**variables)
+    except (KeyError, ValueError):
+        body = template['body']
+    
+    # Clean up empty lines from missing optional variables
+    body = '\n'.join(line for line in body.split('\n') if line.strip())
+    
+    return {
+        'title': title,
+        'body': body,
+        'severity': variables.get('severity', 'INFO'),
+    }
+
+
+def get_event_types_by_group() -> Dict[str, list]:
+    """Get all event types organized by group, for UI rendering.
+    
+    Returns:
+        {group_key: [{'type': event_type, 'title': template_title, 
+                       'default_enabled': bool}, ...]}
+    """
+    result = {}
+    for event_type, template in TEMPLATES.items():
+        group = template.get('group', 'system')
+        if group not in result:
+            result[group] = []
+        result[group].append({
+            'type': event_type,
+            'title': template['title'].replace('{hostname}', '').strip(': '),
+            'default_enabled': template.get('default_enabled', True),
+        })
+    return result
+
+
+def get_default_enabled_events() -> Dict[str, bool]:
+    """Get the default enabled state for all event types."""
+    return {
+        event_type: template.get('default_enabled', True)
+        for event_type, template in TEMPLATES.items()
+    }
+
+
+# ─── AI Enhancement (Optional) ───────────────────────────────────
+
+class AIEnhancer:
+    """Optional AI message enhancement using external LLM API.
+    
+    Enriches template-generated messages with context and suggestions.
+    Falls back to original message if AI is unavailable or fails.
+    """
+    
+    SYSTEM_PROMPT = """You are a Proxmox system administrator assistant. 
+You receive a notification message about a server event and must enhance it with:
+1. A brief explanation of what this means in practical terms
+2. A suggested action if applicable (1-2 sentences max)
+
+Keep the response concise (max 3 sentences total). Do not repeat the original message.
+Respond in the same language as the input message."""
+    
+    def __init__(self, provider: str, api_key: str, model: str = ''):
+        self.provider = provider.lower()
+        self.api_key = api_key
+        self.model = model
+        self._enabled = bool(api_key)
+    
+    @property
+    def enabled(self) -> bool:
+        return self._enabled
+    
+    def enhance(self, title: str, body: str, severity: str) -> Optional[str]:
+        """Enhance a notification message with AI context.
+        
+        Returns enhanced body text, or None if enhancement fails/disabled.
+        """
+        if not self._enabled:
+            return None
+        
+        try:
+            if self.provider in ('openai', 'groq'):
+                return self._call_openai_compatible(title, body, severity)
+        except Exception as e:
+            print(f"[AIEnhancer] Enhancement failed: {e}")
+        
+        return None
+    
+    def _call_openai_compatible(self, title: str, body: str, severity: str) -> Optional[str]:
+        """Call OpenAI-compatible API (works with OpenAI, Groq, local)."""
+        if self.provider == 'groq':
+            url = 'https://api.groq.com/openai/v1/chat/completions'
+            model = self.model or 'llama-3.3-70b-versatile'
+        else:  # openai
+            url = 'https://api.openai.com/v1/chat/completions'
+            model = self.model or 'gpt-4o-mini'
+        
+        user_msg = f"Severity: {severity}\nTitle: {title}\nMessage: {body}"
+        
+        payload = json.dumps({
+            'model': model,
+            'messages': [
+                {'role': 'system', 'content': self.SYSTEM_PROMPT},
+                {'role': 'user', 'content': user_msg},
+            ],
+            'max_tokens': 150,
+            'temperature': 0.3,
+        }).encode('utf-8')
+        
+        headers = {
+            'Content-Type': 'application/json',
+            'Authorization': f'Bearer {self.api_key}',
+        }
+        
+        req = urllib.request.Request(url, data=payload, headers=headers)
+        with urllib.request.urlopen(req, timeout=10) as resp:
+            result = json.loads(resp.read().decode('utf-8'))
+            content = result['choices'][0]['message']['content'].strip()
+            return content if content else None
+
+
+def format_with_ai(title: str, body: str, severity: str,
+                   ai_config: Dict[str, str]) -> str:
+    """Format a message with optional AI enhancement.
+    
+    If AI is configured and succeeds, appends AI insight to the body.
+    Otherwise returns the original body unchanged.
+    
+    Args:
+        title: Notification title
+        body: Notification body
+        severity: Severity level
+        ai_config: {'enabled': 'true', 'provider': 'groq', 'api_key': '...', 'model': ''}
+    
+    Returns:
+        Enhanced body string
+    """
+    if ai_config.get('enabled') != 'true' or not ai_config.get('api_key'):
+        return body
+    
+    enhancer = AIEnhancer(
+        provider=ai_config.get('provider', 'groq'),
+        api_key=ai_config['api_key'],
+        model=ai_config.get('model', ''),
+    )
+    
+    insight = enhancer.enhance(title, body, severity)
+    if insight:
+        return f"{body}\n\n---\n{insight}"
+    
+    return body