feat(oci): watchdog that restarts an application when it stops on its own

- New service that starts again an OCI application that stopped on its own, such as Frigate after Save & Restart. A stop or shutdown asked by the user, a backup, a migration and a ProxMenux operation are never undone.
- The installer asks it in both modes and proposes what the recipe declares; it is changed from the management menu or with the Watchdog switch of the container modal in the Monitor.
- Notifications when the watchdog restarts an application and when it keeps stopping.
- The service is recorded in the change journal, and the feature is documented.

AppImage/scripts/flask_notification_routes.py

@@ -1631,12 +1631,13 @@ _OCI_EVENTS = {
     'oci_update_completed': 'INFO', 'oci_update_failed': 'WARNING',
     'oci_recreate_completed': 'INFO', 'oci_recreate_failed': 'WARNING',
     'oci_modify_completed': 'INFO', 'oci_modify_failed': 'WARNING',
+    'oci_watchdog_restarted': 'WARNING', 'oci_watchdog_failed': 'CRITICAL',
 }
 
 
 @notification_bp.route('/api/internal/oci-event', methods=['POST'])
 def internal_oci_event():
-    """Called by the OCI engine when an update, a change or a recreation ends, with its
+    """Called by the OCI engine when an operation ends or the watchdog acts, with its
     result. Only accepts requests from this host."""
     remote_addr = request.remote_addr or ''
     try:

AppImage/scripts/flask_server.py

@@ -15438,6 +15438,39 @@ def api_lxc_oci_instance(vmid):
         return jsonify({"ok": False, "error": str(e)}), 500
 
 
+@app.route('/api/lxc/<int:vmid>/oci-watchdog', methods=['POST'])
+@require_admin_scope
+def api_lxc_oci_watchdog(vmid):
+    """Turn the watchdog of an OCI application on or off: whether it is
+    started again when it stops on its own. The OCI engine keeps the choice
+    in the record of every container of the application.
+
+    Body: {"enabled": true|false}
+    """
+    data = request.get_json(silent=True) or {}
+    enabled = data.get('enabled')
+    if not isinstance(enabled, bool):
+        return jsonify({'error': 'enabled must be true or false'}), 400
+    try:
+        import oci_instance_info
+        if not oci_instance_info.info(vmid).get('oci_instance'):
+            return jsonify({'error': 'not an OCI application'}), 404
+        result = subprocess.run(
+            [sys.executable, '/usr/local/share/proxmenux/oci/engine/remote/oci_watchdog.py',
+             'set', str(int(vmid)), 'on' if enabled else 'off'],
+            capture_output=True, text=True, timeout=60,
+            env={k: v for k, v in os.environ.items() if k not in ('PYTHONPATH', 'LD_LIBRARY_PATH')})
+        if result.returncode == 3:
+            return jsonify({'error': 'busy', 'detail': 'Another OCI operation is in progress'}), 409
+        if result.returncode != 0:
+            return jsonify({'error': (result.stderr or result.stdout).strip()[-300:] or 'failed'}), 500
+        return jsonify({'ok': True, 'watchdog': enabled, **oci_instance_info.info(vmid)})
+    except subprocess.TimeoutExpired:
+        return jsonify({'error': 'timeout'}), 504
+    except Exception as e:
+        return jsonify({'ok': False, 'error': str(e)}), 500
+
+
 @app.route('/api/vms/<int:vmid>/logs', methods=['GET'])
 @require_auth
 def api_vm_logs(vmid):

AppImage/scripts/notification_templates.py

@@ -927,6 +927,24 @@ TEMPLATES = {
         'group': 'vm_ct',
         'default_enabled': True,
     },
+    'oci_watchdog_restarted': {
+        'title': '{hostname}: {app_name} restarted after it stopped',
+        'body': '{app_name} stopped on its own and the watchdog started it again.\nContainers: {containers}',
+        'label': 'OCI application restarted by the watchdog',
+        'group': 'vm_ct',
+        'default_enabled': True,
+    },
+    'oci_watchdog_failed': {
+        'title': '{hostname}: {app_name} keeps stopping',
+        'body': (
+            '{app_name} keeps stopping on its own and the watchdog no longer restarts it.\n'
+            'Containers: {containers}\n'
+            'Start it by hand once the cause is solved; the watchdog then watches it again.'
+        ),
+        'label': 'OCI application keeps stopping',
+        'group': 'vm_ct',
+        'default_enabled': True,
+    },
     'app_update_available': {
         'title': '{hostname}: {app_name} update available on CT {vmid}',
         'body': (
@@ -2377,6 +2395,8 @@ EVENT_EMOJI = {
     'oci_recreate_failed':    '\u26A0\uFE0F',
     'oci_modify_completed':   '\u2705',
     'oci_modify_failed':      '\u26A0\uFE0F',
+    'oci_watchdog_restarted': '\U0001F504',
+    'oci_watchdog_failed':    '\u26A0\uFE0F',
     'app_update_available': '\U0001F195',  # \ud83c\udd95 NEW \u2014 upstream app release
     'docker_stack_update_available': '\U0001F433',
     'vm_start':             '\u25B6\uFE0F',    # play button

AppImage/scripts/oci_instance_info.py

@@ -3,8 +3,9 @@
 Read-only view of the installation record OCI manager Apps keeps for every
 container it created: whether the container is one, whether it belongs to a
 multi-container application, whether it uses host directories (which its
-backup does not revert), whether an operation is pending and whether it was
-restored from a backup and is not registered on this host yet. Nothing here
+backup does not revert), whether an operation is pending, whether it is
+restarted when it stops on its own and whether it was restored from a backup
+and is not registered on this host yet. Nothing here
 changes the record or runs inside the container.
 """
 from __future__ import annotations
@@ -70,6 +71,7 @@ def info(vmid: int) -> dict:
         "host_directories": False,
         "pending": False,
         "restored": False,
+        "watchdog": False,
     }
     record = _record(vmid)
     installation = _installation(vmid)
@@ -91,5 +93,6 @@ def info(vmid: int) -> dict:
         members=members,
         host_directories=any(_host_dirs(r) for r in records),
         pending=bool(record.get("pending_transaction") or primary.get("pending_stack_transaction")),
+        watchdog=all((r.get("deployment") or {}).get("watchdog") is True for r in records),
     )
     return result