feat(oci): watchdog that restarts an application when it stops on its own

- New service that starts again an OCI application that stopped on its own, such as Frigate after Save & Restart. A stop or shutdown asked by the user, a backup, a migration and a ProxMenux operation are never undone. - The installer asks it in both modes and proposes what the recipe declares; it is changed from the management menu or with the Watchdog switch of the container modal in the Monitor. - Notifications when the watchdog restarts an application and when it keeps stopping. - The service is recorded in the change journal, and the feature is documented.
2026-10-09 06:56:37 +00:00 · 2026-10-06 23:18:26 +02:00
parent 6b70c3934e
commit 41ae752da1
28 changed files with 1006 additions and 19 deletions
@@ -52,6 +52,7 @@ interface OciInstanceInfo {
  host_directories: boolean
  pending: boolean
  restored?: boolean
+  watchdog?: boolean
 }

 interface LxcUpdateCheck {
@@ -887,6 +888,8 @@ export function VirtualMachines() {
  //   savedOnboot       → 2 s ack pill after successful save.
  const [resourcesEditMode, setResourcesEditMode] = useState(false)
  const [pendingOnboot, setPendingOnboot] = useState<boolean | null>(null)
+  // Watchdog of an OCI application: same edit-gate as the toggle above.
+  const [pendingWatchdog, setPendingWatchdog] = useState<boolean | null>(null)
  const [pendingTags, setPendingTags] = useState<string[] | null>(null)
  const [newTagDraft, setNewTagDraft] = useState<string>("")
  // When set (in edit mode), the pill at this index renders as an
@@ -1251,6 +1254,7 @@ export function VirtualMachines() {
    // start in view mode with no pending change.
    setResourcesEditMode(false)
    setPendingOnboot(null)
+    setPendingWatchdog(null)
    setPendingTags(null)
    setNewTagDraft("")
    setEditingTagIndex(null)
@@ -1682,6 +1686,7 @@ export function VirtualMachines() {

  const handleCancelResourcesEdit = () => {
    setPendingOnboot(null)
+    setPendingWatchdog(null)
    setPendingTags(null)
    setNewTagDraft("")
    setEditingTagIndex(null)
@@ -1702,16 +1707,26 @@ export function VirtualMachines() {
    if (pendingTags !== null && stringifyTags(pendingTags) !== stringifyTags(currentTags)) {
      payload.tags = pendingTags
    }
-    if (Object.keys(payload).length === 0) {
+    const watchdogChanged = !!ociInstance?.oci_instance && pendingWatchdog !== null && pendingWatchdog !== !!ociInstance.watchdog
+    if (Object.keys(payload).length === 0 && !watchdogChanged) {
      handleCancelResourcesEdit()
      return
    }
    setSavingOnboot(true)
    try {
-      await fetchApi(`/api/vms/${selectedVM.vmid}/config`, {
-        method: "POST",
-        body: JSON.stringify(payload),
-      })
+      if (Object.keys(payload).length > 0) {
+        await fetchApi(`/api/vms/${selectedVM.vmid}/config`, {
+          method: "POST",
+          body: JSON.stringify(payload),
+        })
+      }
+      if (watchdogChanged) {
+        const updated = await fetchApi<OciInstanceInfo>(`/api/lxc/${selectedVM.vmid}/oci-watchdog`, {
+          method: "POST",
+          body: JSON.stringify({ enabled: pendingWatchdog }),
+        })
+        setOciInstance((prev) => (prev ? { ...prev, watchdog: !!updated?.watchdog } : prev))
+      }
      // Optimistic local reflect so the UI doesn't wait a poll cycle.
      if (payload.onboot !== undefined) {
        setVMDetails((prev) => (prev ? { ...prev, config: { ...prev.config, onboot: payload.onboot as number } } : prev))
@@ -1725,6 +1740,7 @@ export function VirtualMachines() {
      // list card too without waiting up to 2.5 s.
      void mutate()
      setPendingOnboot(null)
+      setPendingWatchdog(null)
      setPendingTags(null)
      setNewTagDraft("")
      setEditingTagIndex(null)
@@ -4091,6 +4107,7 @@ const handleDownloadLogs = async (vmid: number, vmName: string) => {
                                          savingOnboot ||
                                          (
                                            (pendingOnboot === null || pendingOnboot === !!vmDetails.config.onboot) &&
+                                            (pendingWatchdog === null || !ociInstance?.oci_instance || pendingWatchdog === !!ociInstance.watchdog) &&
                                            (pendingTags === null || stringifyTags(pendingTags) === stringifyTags(parseTags(selectedVM?.tags)))
                                          )
                                        }
@@ -4323,6 +4340,34 @@ const handleDownloadLogs = async (vmid: number, vmName: string) => {
                                />
                              </div>

+                              {/* Watchdog of an OCI application: started
+                                  again when it stops on its own. */}
+                              {ociInstance?.oci_instance && (
+                                <div
+                                  className={`mt-1 rounded-md p-3 flex items-center justify-between gap-3 ${
+                                    resourcesEditMode ? "bg-accent" : ""
+                                  }`}
+                                >
+                                  <div className="flex items-start gap-2 min-w-0">
+                                    <Eye className="h-4 w-4 text-blue-500 shrink-0 mt-0.5" />
+                                    <div className="min-w-0">
+                                      <div className="text-sm font-medium text-foreground">
+                                        {t("vmLxc.details.watchdog")}
+                                      </div>
+                                      <div className="text-xs text-muted-foreground leading-relaxed">
+                                        {t("vmLxc.details.watchdogHelp")}
+                                      </div>
+                                    </div>
+                                  </div>
+                                  <Switch
+                                    checked={pendingWatchdog ?? !!ociInstance.watchdog}
+                                    disabled={!resourcesEditMode || savingOnboot}
+                                    onCheckedChange={(v) => setPendingWatchdog(v)}
+                                    className={`data-[state=checked]:bg-blue-600 data-[state=unchecked]:bg-input border border-border ${!resourcesEditMode ? "opacity-60" : ""}`}
+                                  />
+                                </div>
+                              )}
+
                              {/* IP Addresses with proper keys */}
                              {selectedVM?.type === "lxc" && vmDetails?.lxc_ip_info && (
                                <div className="mt-4 lg:mt-6">
@@ -1049,6 +1049,8 @@
        "mounted": "montiert"
      },
      "startOnBoot": "Beim Booten beginnen",
+      "watchdog": "Watchdog",
+      "watchdogHelp": "Startet diese Anwendung automatisch neu, wenn sie abstürzt.",
      "tags": "Tags",
      "tagsPlaceholder": "Tag hinzufügen…",
      "tagsNone": "Keine Tags",
@@ -2008,6 +2010,8 @@
        "oci_recreate_failed": "Neuerstellung der OCI-Anwendung fehlgeschlagen",
        "oci_modify_completed": "OCI-Anwendung geändert",
        "oci_modify_failed": "Änderung der OCI-Anwendung fehlgeschlagen",
+        "oci_watchdog_restarted": "OCI-Anwendung vom Watchdog neu gestartet",
+        "oci_watchdog_failed": "OCI-Anwendung stoppt immer wieder",
        "app_update_available": "App-Update verfügbar",
        "lxc_update_applied": "LXC Update angewendet",
        "docker_stack_update_available": "Docker Updates verfügbar"
@@ -6441,6 +6445,16 @@
          "body": "Die Änderung von {app_name} wurde nicht abgeschlossen.\nGrund: {reason}\nContainer: {containers}\nÖffnen Sie „Installierte OCI-Anwendungen verwalten“, um den Zustand zu prüfen.",
          "label": "Änderung der OCI-Anwendung fehlgeschlagen"
        },
+        "oci_watchdog_restarted": {
+          "title": "{hostname}: {app_name} nach einem Stopp neu gestartet",
+          "body": "{app_name} wurde von selbst beendet und vom Watchdog neu gestartet.\nContainer: {containers}",
+          "label": "OCI-Anwendung vom Watchdog neu gestartet"
+        },
+        "oci_watchdog_failed": {
+          "title": "{hostname}: {app_name} stoppt immer wieder",
+          "body": "{app_name} stoppt immer wieder von selbst und wird vom Watchdog nicht mehr neu gestartet.\nContainer: {containers}\nStarten Sie die Anwendung von Hand, sobald die Ursache behoben ist; der Watchdog überwacht sie dann wieder.",
+          "label": "OCI-Anwendung stoppt immer wieder"
+        },
        "app_update_available": {
          "title": "{hostname}: {app_name} Update verfügbar auf CT {vmid}",
          "body": "{app_name} auf CT {vmid} ({ct_name}) hat eine neue Version:\n    {installed} → {latest}",
@@ -1049,6 +1049,8 @@
        "mounted": "montado"
      },
      "startOnBoot": "Iniciar al arrancar",
+      "watchdog": "Vigilancia",
+      "watchdogHelp": "Reinicia automáticamente esta aplicación cuando falla.",
      "tags": "Etiquetas",
      "tagsPlaceholder": "Añadir etiqueta…",
      "tagsNone": "Sin etiquetas",
@@ -1995,6 +1997,8 @@
        "oci_recreate_failed": "Recreación de aplicación OCI fallida",
        "oci_modify_completed": "Aplicación OCI modificada",
        "oci_modify_failed": "Modificación de aplicación OCI fallida",
+        "oci_watchdog_restarted": "Aplicación OCI reiniciada por la vigilancia",
+        "oci_watchdog_failed": "Aplicación OCI que se detiene una y otra vez",
        "app_update_available": "Actualización de app disponible",
        "ai_model_migrated": "Modelo de IA actualizado automáticamente",
        "proxmenux_update": "Actualización de ProxMenux disponible",
@@ -6441,6 +6445,16 @@
          "body": "La modificación de {app_name} no se completó.\nMotivo: {reason}\nContenedores: {containers}\nAbre Gestionar aplicaciones OCI instaladas para comprobar su estado.",
          "label": "Modificación de aplicación OCI fallida"
        },
+        "oci_watchdog_restarted": {
+          "title": "{hostname}: {app_name} reiniciado tras detenerse",
+          "body": "{app_name} se detuvo solo y la vigilancia lo ha iniciado de nuevo.\nContenedores: {containers}",
+          "label": "Aplicación OCI reiniciada por la vigilancia"
+        },
+        "oci_watchdog_failed": {
+          "title": "{hostname}: {app_name} se detiene una y otra vez",
+          "body": "{app_name} sigue deteniéndose solo y la vigilancia ya no lo reinicia.\nContenedores: {containers}\nInícialo a mano cuando la causa esté resuelta; la vigilancia volverá a vigilarlo.",
+          "label": "Aplicación OCI que se detiene una y otra vez"
+        },
        "app_update_available": {
          "title": "{hostname}: actualización de {app_name} disponible en CT {vmid}",
          "body": "{app_name} en CT {vmid} ({ct_name}) tiene una nueva versión:\n    {installed} → {latest}",
@@ -1049,6 +1049,8 @@
        "mounted": "monté"
      },
      "startOnBoot": "Démarrer au démarrage",
+      "watchdog": "Surveillance",
+      "watchdogHelp": "Redémarre automatiquement cette application lorsqu'elle plante.",
      "tags": "balises",
      "tagsPlaceholder": "Ajouter une balise…",
      "tagsNone": "Aucune balise",
@@ -2008,6 +2010,8 @@
        "oci_recreate_failed": "Échec de la recréation de l'application OCI",
        "oci_modify_completed": "Application OCI modifiée",
        "oci_modify_failed": "Échec de la modification de l'application OCI",
+        "oci_watchdog_restarted": "Application OCI redémarrée par la surveillance",
+        "oci_watchdog_failed": "Application OCI qui s'arrête sans cesse",
        "app_update_available": "mise à jour de l'application disponible",
        "lxc_update_applied": "Mise à jour LXC appliquée",
        "docker_stack_update_available": "Docker mises à jour disponibles"
@@ -6441,6 +6445,16 @@
          "body": "La modification de {app_name} n'a pas abouti.\nMotif : {reason}\nConteneurs : {containers}\nOuvrez Gérer les applications OCI installées pour vérifier son état.",
          "label": "Échec de la modification de l'application OCI"
        },
+        "oci_watchdog_restarted": {
+          "title": "{hostname} : {app_name} redémarré après un arrêt",
+          "body": "{app_name} s'est arrêté tout seul et la surveillance l'a redémarré.\nConteneurs : {containers}",
+          "label": "Application OCI redémarrée par la surveillance"
+        },
+        "oci_watchdog_failed": {
+          "title": "{hostname} : {app_name} s'arrête sans cesse",
+          "body": "{app_name} continue de s'arrêter tout seul et la surveillance ne le redémarre plus.\nConteneurs : {containers}\nDémarrez-le à la main une fois la cause résolue ; la surveillance le reprend alors.",
+          "label": "Application OCI qui s'arrête sans cesse"
+        },
        "app_update_available": {
          "title": "{hostname} : mise à jour {app_name} disponible sur CT {vmid}",
          "body": "{app_name} sur CT {vmid} ({ct_name}) a une nouvelle version :\n    {installed} → {latest}",
@@ -1049,6 +1049,8 @@
        "mounted": "montato"
      },
      "startOnBoot": "Avvio al boot",
+      "watchdog": "Watchdog",
+      "watchdogHelp": "Riavvia automaticamente questa applicazione quando si arresta in modo anomalo.",
      "tags": "Tag",
      "tagsPlaceholder": "Aggiungi tag…",
      "tagsNone": "Nessun tag",
@@ -2008,6 +2010,8 @@
        "oci_recreate_failed": "Ricreazione app OCI non riuscita",
        "oci_modify_completed": "App OCI modificata",
        "oci_modify_failed": "Modifica app OCI non riuscita",
+        "oci_watchdog_restarted": "App OCI riavviata dal watchdog",
+        "oci_watchdog_failed": "App OCI che continua ad arrestarsi",
        "app_update_available": "Aggiornamento app disponibile",
        "lxc_update_applied": "Aggiornamento LXC applicato",
        "docker_stack_update_available": "Aggiornamenti Docker disponibili"
@@ -6441,6 +6445,16 @@
          "body": "La modifica di {app_name} non è stata completata.\nMotivo: {reason}\nContainer: {containers}\nApri «Gestisci le applicazioni OCI installate» per verificarne lo stato.",
          "label": "Modifica app OCI non riuscita"
        },
+        "oci_watchdog_restarted": {
+          "title": "{hostname}: {app_name} riavviato dopo un arresto",
+          "body": "{app_name} si è arrestato da solo e il watchdog lo ha riavviato.\nContainer: {containers}",
+          "label": "App OCI riavviata dal watchdog"
+        },
+        "oci_watchdog_failed": {
+          "title": "{hostname}: {app_name} continua ad arrestarsi",
+          "body": "{app_name} continua ad arrestarsi da solo e il watchdog non lo riavvia più.\nContainer: {containers}\nAvvialo a mano una volta risolta la causa; il watchdog tornerà a sorvegliarlo.",
+          "label": "App OCI che continua ad arrestarsi"
+        },
        "app_update_available": {
          "title": "{hostname}: aggiornamento {app_name} disponibile su CT {vmid}",
          "body": "{app_name} su CT {vmid} ({ct_name}) ha una nuova versione:\n    {installed} → {latest}",
@@ -1049,6 +1049,8 @@
        "mounted": "montado"
      },
      "startOnBoot": "Iniciar na inicialização",
+      "watchdog": "Vigilância",
+      "watchdogHelp": "Reinicia automaticamente esta aplicação quando falha.",
      "tags": "Etiquetas",
      "tagsPlaceholder": "Adicionar tag…",
      "tagsNone": "sem tags",
@@ -2008,6 +2010,8 @@
        "oci_recreate_failed": "Falha na recriação da aplicação OCI",
        "oci_modify_completed": "Aplicação OCI modificada",
        "oci_modify_failed": "Falha na modificação da aplicação OCI",
+        "oci_watchdog_restarted": "Aplicação OCI reiniciada pela vigilância",
+        "oci_watchdog_failed": "Aplicação OCI que continua a parar",
        "app_update_available": "atualização de aplicativo disponível",
        "lxc_update_applied": "Atualização LXC aplicada",
        "docker_stack_update_available": "Docker atualizações disponíveis"
@@ -6441,6 +6445,16 @@
          "body": "A modificação de {app_name} não foi concluída.\nMotivo: {reason}\nContêineres: {containers}\nAbra Gerir aplicações OCI instaladas para verificar o estado.",
          "label": "Falha na modificação da aplicação OCI"
        },
+        "oci_watchdog_restarted": {
+          "title": "{hostname}: {app_name} reiniciado depois de parar",
+          "body": "{app_name} parou sozinho e a vigilância voltou a iniciá-lo.\nContêineres: {containers}",
+          "label": "Aplicação OCI reiniciada pela vigilância"
+        },
+        "oci_watchdog_failed": {
+          "title": "{hostname}: {app_name} continua a parar",
+          "body": "{app_name} continua a parar sozinho e a vigilância já não o reinicia.\nContêineres: {containers}\nInicie-o à mão quando a causa estiver resolvida; a vigilância volta então a vigiá-lo.",
+          "label": "Aplicação OCI que continua a parar"
+        },
        "app_update_available": {
          "title": "{hostname}: atualização {app_name} disponível no CT {vmid}",
          "body": "{app_name} no CT {vmid} ({ct_name}) tem uma nova versão:\n    {installed} → {latest}",
@@ -1073,6 +1073,8 @@
        "configuredButNotMounted": "Nastavené, ale nepripojené"
      },
      "startOnBoot": "Spúšťať pri štarte",
+      "watchdog": "Watchdog",
+      "watchdogHelp": "Automaticky reštartuje túto aplikáciu, keď zlyhá.",
      "tags": "Značky",
      "tagsPlaceholder": "Pridať značku…",
      "tagsNone": "Žiadne značky"
@@ -2009,6 +2011,8 @@
        "oci_recreate_failed": "Opätovné vytvorenie OCI aplikácie zlyhalo",
        "oci_modify_completed": "OCI aplikácia upravená",
        "oci_modify_failed": "Úprava OCI aplikácie zlyhala",
+        "oci_watchdog_restarted": "OCI aplikácia reštartovaná watchdogom",
+        "oci_watchdog_failed": "OCI aplikácia sa opakovane zastavuje",
        "app_update_available": "K dispozícii je aktualizácia aplikácie"
      },
      "ui": {
@@ -6440,6 +6444,16 @@
          "body": "Úprava aplikácie {app_name} sa nedokončila.\nDôvod: {reason}\nKontajnery: {containers}\nOtvorte Spravovať nainštalované OCI aplikácie a skontrolujte jej stav.",
          "label": "Úprava OCI aplikácie zlyhala"
        },
+        "oci_watchdog_restarted": {
+          "title": "{hostname}: {app_name} reštartovaná po zastavení",
+          "body": "Aplikácia {app_name} sa sama zastavila a watchdog ju znova spustil.\nKontajnery: {containers}",
+          "label": "OCI aplikácia reštartovaná watchdogom"
+        },
+        "oci_watchdog_failed": {
+          "title": "{hostname}: {app_name} sa opakovane zastavuje",
+          "body": "Aplikácia {app_name} sa stále sama zastavuje a watchdog ju už nereštartuje.\nKontajnery: {containers}\nPo odstránení príčiny ju spustite ručne; watchdog ju potom bude znova sledovať.",
+          "label": "OCI aplikácia sa opakovane zastavuje"
+        },
        "app_update_available": {
          "title": "{hostname}: Pre {app_name} je na CT {vmid} dostupná aktualizácia",
          "body": "Aplikácia {app_name} na CT {vmid} ({ct_name}) má novú verziu:\n    {installed} → {latest}",
@@ -1049,6 +1049,8 @@
        "mounted": "monterad"
      },
      "startOnBoot": "Börja vid start",
+      "watchdog": "Watchdog",
+      "watchdogHelp": "Startar om den här applikationen automatiskt när den kraschar.",
      "tags": "Taggar",
      "tagsPlaceholder": "Lägg till tagg...",
      "tagsNone": "Inga taggar",
@@ -2008,6 +2010,8 @@
        "oci_recreate_failed": "Återskapande av OCI-applikation misslyckades",
        "oci_modify_completed": "OCI-applikation ändrad",
        "oci_modify_failed": "Ändring av OCI-applikation misslyckades",
+        "oci_watchdog_restarted": "OCI-applikation omstartad av watchdog",
+        "oci_watchdog_failed": "OCI-applikation stannar om och om igen",
        "app_update_available": "Appuppdatering tillgänglig",
        "lxc_update_applied": "LXC uppdatering tillämpad",
        "docker_stack_update_available": "Docker uppdateringar tillgängliga"
@@ -6441,6 +6445,16 @@
          "body": "Ändringen av {app_name} slutfördes inte.\nOrsak: {reason}\nContainrar: {containers}\nÖppna Hantera installerade OCI-applikationer för att kontrollera dess tillstånd.",
          "label": "Ändring av OCI-applikation misslyckades"
        },
+        "oci_watchdog_restarted": {
+          "title": "{hostname}: {app_name} omstartad efter att ha stannat",
+          "body": "{app_name} stannade av sig själv och watchdog startade den igen.\nContainrar: {containers}",
+          "label": "OCI-applikation omstartad av watchdog"
+        },
+        "oci_watchdog_failed": {
+          "title": "{hostname}: {app_name} stannar om och om igen",
+          "body": "{app_name} fortsätter att stanna av sig själv och watchdog startar inte längre om den.\nContainrar: {containers}\nStarta den manuellt när orsaken är löst; watchdog övervakar den sedan igen.",
+          "label": "OCI-applikation stannar om och om igen"
+        },
        "app_update_available": {
          "title": "{hostname}: {app_name} uppdatering tillgänglig på CT {vmid}",
          "body": "{app_name} på CT {vmid} ({ct_name}) har en ny version:\n    {installed} → {latest}",
@@ -1631,12 +1631,13 @@ _OCI_EVENTS = {
    'oci_update_completed': 'INFO', 'oci_update_failed': 'WARNING',
    'oci_recreate_completed': 'INFO', 'oci_recreate_failed': 'WARNING',
    'oci_modify_completed': 'INFO', 'oci_modify_failed': 'WARNING',
+    'oci_watchdog_restarted': 'WARNING', 'oci_watchdog_failed': 'CRITICAL',
 }


@notification_bp.route('/api/internal/oci-event', methods=['POST'])
 def internal_oci_event():
-    """Called by the OCI engine when an update, a change or a recreation ends, with its
+    """Called by the OCI engine when an operation ends or the watchdog acts, with its
    result. Only accepts requests from this host."""
    remote_addr = request.remote_addr or ''
    try:
@@ -15438,6 +15438,39 @@ def api_lxc_oci_instance(vmid):
        return jsonify({"ok": False, "error": str(e)}), 500


+@app.route('/api/lxc/<int:vmid>/oci-watchdog', methods=['POST'])
+@require_admin_scope
+def api_lxc_oci_watchdog(vmid):
+    """Turn the watchdog of an OCI application on or off: whether it is
+    started again when it stops on its own. The OCI engine keeps the choice
+    in the record of every container of the application.
+
+    Body: {"enabled": true|false}
+    """
+    data = request.get_json(silent=True) or {}
+    enabled = data.get('enabled')
+    if not isinstance(enabled, bool):
+        return jsonify({'error': 'enabled must be true or false'}), 400
+    try:
+        import oci_instance_info
+        if not oci_instance_info.info(vmid).get('oci_instance'):
+            return jsonify({'error': 'not an OCI application'}), 404
+        result = subprocess.run(
+            [sys.executable, '/usr/local/share/proxmenux/oci/engine/remote/oci_watchdog.py',
+             'set', str(int(vmid)), 'on' if enabled else 'off'],
+            capture_output=True, text=True, timeout=60,
+            env={k: v for k, v in os.environ.items() if k not in ('PYTHONPATH', 'LD_LIBRARY_PATH')})
+        if result.returncode == 3:
+            return jsonify({'error': 'busy', 'detail': 'Another OCI operation is in progress'}), 409
+        if result.returncode != 0:
+            return jsonify({'error': (result.stderr or result.stdout).strip()[-300:] or 'failed'}), 500
+        return jsonify({'ok': True, 'watchdog': enabled, **oci_instance_info.info(vmid)})
+    except subprocess.TimeoutExpired:
+        return jsonify({'error': 'timeout'}), 504
+    except Exception as e:
+        return jsonify({'ok': False, 'error': str(e)}), 500
+
+
@app.route('/api/vms/<int:vmid>/logs', methods=['GET'])
@require_auth
 def api_vm_logs(vmid):
@@ -927,6 +927,24 @@ TEMPLATES = {
        'group': 'vm_ct',
        'default_enabled': True,
    },
+    'oci_watchdog_restarted': {
+        'title': '{hostname}: {app_name} restarted after it stopped',
+        'body': '{app_name} stopped on its own and the watchdog started it again.\nContainers: {containers}',
+        'label': 'OCI application restarted by the watchdog',
+        'group': 'vm_ct',
+        'default_enabled': True,
+    },
+    'oci_watchdog_failed': {
+        'title': '{hostname}: {app_name} keeps stopping',
+        'body': (
+            '{app_name} keeps stopping on its own and the watchdog no longer restarts it.\n'
+            'Containers: {containers}\n'
+            'Start it by hand once the cause is solved; the watchdog then watches it again.'
+        ),
+        'label': 'OCI application keeps stopping',
+        'group': 'vm_ct',
+        'default_enabled': True,
+    },
    'app_update_available': {
        'title': '{hostname}: {app_name} update available on CT {vmid}',
        'body': (
@@ -2377,6 +2395,8 @@ EVENT_EMOJI = {
    'oci_recreate_failed':    '\u26A0\uFE0F',
    'oci_modify_completed':   '\u2705',
    'oci_modify_failed':      '\u26A0\uFE0F',
+    'oci_watchdog_restarted': '\U0001F504',
+    'oci_watchdog_failed':    '\u26A0\uFE0F',
    'app_update_available': '\U0001F195',  # \ud83c\udd95 NEW \u2014 upstream app release
    'docker_stack_update_available': '\U0001F433',
    'vm_start':             '\u25B6\uFE0F',    # play button
@@ -3,8 +3,9 @@
 Read-only view of the installation record OCI manager Apps keeps for every
 container it created: whether the container is one, whether it belongs to a
 multi-container application, whether it uses host directories (which its
-backup does not revert), whether an operation is pending and whether it was
-restored from a backup and is not registered on this host yet. Nothing here
+backup does not revert), whether an operation is pending, whether it is
+restarted when it stops on its own and whether it was restored from a backup
+and is not registered on this host yet. Nothing here
 changes the record or runs inside the container.
 """
 from __future__ import annotations
@@ -70,6 +71,7 @@ def info(vmid: int) -> dict:
        "host_directories": False,
        "pending": False,
        "restored": False,
+        "watchdog": False,
    }
    record = _record(vmid)
    installation = _installation(vmid)
@@ -91,5 +93,6 @@ def info(vmid: int) -> dict:
        members=members,
        host_directories=any(_host_dirs(r) for r in records),
        pending=bool(record.get("pending_transaction") or primary.get("pending_stack_transaction")),
+        watchdog=all((r.get("deployment") or {}).get("watchdog") is True for r in records),
    )
    return result