fix(oci): keep every declared image data path on a volume so installs can be updated

.github/scripts/tests/test_oci_update_preservation_wording.py

@@ -21,10 +21,10 @@ CONFIRM = ('Apply options from the current catalog template? New required paths
            'Review the resulting configuration before recreating the CT.')
 OLD_GUARD = 'The current template changes the image or identity; an explicit migration is required'
 GUARD = 'The current template changes the template identity or image repository; an explicit migration is required'
-OLD_PREVIEW = ('The current image of the saved channel will be checked and downloaded. Resources, paths and GPU are kept. '
-               'The CT is stopped during the replacement and a native backup is created first.')
-PREVIEW = ('The saved image channel is checked for a newer image. If replacement is needed, the CT is stopped and '
-           'a native backup is verified before its root is replaced. Host directories are outside that backup.')
+OLD_PREVIEW = ('The saved image channel is checked for a newer image. If replacement is needed, the CT is stopped and '
+               'a native backup is verified before its root is replaced. Host directories are outside that backup.')
+PREVIEW = ('If the image channel has a new version, the CT is stopped, backed up and verified, and replaced with '
+           'the new image to update the container. If anything fails, the backup is restored.')
 
 
 def extracted(path, name, scope):

lang/es.json

@@ -2741,6 +2741,7 @@
   "If pvesm path returned a DIRECTORY (ZFS/BTRFS):": "Si pvesm path devolvió un DIRECTORIO (ZFS/BTRFS):",
   "If repositories error:": "Si hay error en los repositorios:",
   "If the host freezes, remove hostpci entries from": "Si el host se congela, elimine las entradas hostpci de",
+  "If the image channel has a new version, the CT is stopped, backed up and verified, and replaced with the new image to update the container. If anything fails, the backup is restored.": "Si el canal de imagen tiene una versión nueva, el CT se detiene, se hace un backup, se verifica y se sustituye por la nueva imagen, actualizando el contenedor. Si algo falla, se restaura el backup.",
   "If this happens, you can restore the backup from the 'Subscription Banner Removal' option in 'Uninstall optimizations'.": "Si esto sucede, puede restaurar la copia de seguridad desde la opción 'Eliminación del banner de suscripción' en 'Optimizaciones de desinstalación'.",
   "If this node runs hyper-converged Ceph: ensure Ceph is 19.x (Squid) BEFORE upgrading PVE.": "Si este nodo ejecuta Ceph hiperconvergente: asegúrese de que Ceph sea 19.x (Squid) ANTES de actualizar PVE.",
   "If upgrade fails:": "Si la actualización falla:",

oci/catalog/apps/archivebox.json

@@ -557,6 +557,9 @@
           "owner_strategy": "mapped-root",
           "only_when_mount_type": "managed-volume"
         }
+      ],
+      "non_persistent_image_volumes": [
+        "/out"
       ]
     },
     "adaptations": [

oci/catalog/apps/cloudflared.json

@@ -74,6 +74,22 @@
           "backup": true,
           "default_size_gb": 4
         }
+      },
+      {
+        "id": "volume-1",
+        "container_path": "/root/.cloudflared",
+        "compose_source_example": "/DATA/AppData/$AppID/cloudflared",
+        "read_only": false,
+        "required": true,
+        "installation_choice": [
+          "managed-volume",
+          "host-bind"
+        ],
+        "default": "managed-volume",
+        "managed_volume": {
+          "backup": true,
+          "default_size_gb": 4
+        }
       }
     ],
     "ports": [
@@ -122,6 +138,11 @@
               "type": "bind",
               "source": "/DATA/AppData/cloudflared-cloudflared/config",
               "target": "/config"
+            },
+            {
+              "type": "bind",
+              "source": "/DATA/AppData/$AppID/cloudflared",
+              "target": "/root/.cloudflared"
             }
           ],
           "container_name": "cloudflared"
@@ -151,6 +172,17 @@
         "shared_with_other_lxc": false,
         "source_path": null,
         "source_path_prompt": null
+      },
+      {
+        "id": "cloudflared-volume-1",
+        "service": "cloudflared",
+        "container_path": "/root/.cloudflared",
+        "mode": "managed-volume",
+        "user_selectable": false,
+        "backup": true,
+        "shared_with_other_lxc": false,
+        "source_path": null,
+        "source_path_prompt": null
       }
     ],
     "orchestration": {
@@ -341,6 +373,12 @@
           "remove_lost_found": true,
           "owner_strategy": "mapped-root",
           "only_when_mount_type": "managed-volume"
+        },
+        {
+          "container_path": "/root/.cloudflared",
+          "remove_lost_found": true,
+          "owner_strategy": "mapped-root",
+          "only_when_mount_type": "managed-volume"
         }
       ]
     }

oci/catalog/apps/flaresolverr.json

@@ -267,7 +267,10 @@
       },
       "resources": {
         "cpu_shares": 50
-      }
+      },
+      "non_persistent_image_volumes": [
+        "/config"
+      ]
     },
     "adaptations": [
       {

oci/catalog/apps/handbrake-jlesage.json

@@ -131,6 +131,22 @@
           "default_size_gb": 8
         }
       },
+      {
+        "id": "volume-4",
+        "container_path": "/trash",
+        "compose_source_example": "/DATA/AppData/$AppID/trash",
+        "read_only": false,
+        "required": true,
+        "installation_choice": [
+          "managed-volume",
+          "host-bind"
+        ],
+        "default": "managed-volume",
+        "managed_volume": {
+          "backup": true,
+          "default_size_gb": 8
+        }
+      },
       {
         "id": "volume-3",
         "container_path": "/config",
@@ -206,6 +222,11 @@
               "source": "/DATA/Media",
               "target": "/output"
             },
+            {
+              "type": "bind",
+              "source": "/DATA/AppData/$AppID/trash",
+              "target": "/trash"
+            },
             {
               "type": "bind",
               "source": "/DATA/AppData/$AppID/config",
@@ -262,6 +283,17 @@
         "source_path": null,
         "source_path_prompt": null
       },
+      {
+        "id": "handbrake-volume-4",
+        "service": "handbrake",
+        "container_path": "/trash",
+        "mode": "managed-volume",
+        "user_selectable": false,
+        "backup": true,
+        "shared_with_other_lxc": false,
+        "source_path": null,
+        "source_path_prompt": null
+      },
       {
         "id": "handbrake-volume-3",
         "service": "handbrake",
@@ -395,6 +427,12 @@
           "remove_lost_found": true,
           "owner_strategy": "mapped-application-user",
           "only_when_mount_type": "managed-volume"
+        },
+        {
+          "container_path": "/trash",
+          "remove_lost_found": true,
+          "owner_strategy": "mapped-application-user",
+          "only_when_mount_type": "managed-volume"
         }
       ],
       "startup_healthcheck": {

oci/catalog/apps/medusa-official.json

@@ -114,6 +114,22 @@
           "backup": true,
           "default_size_gb": 8
         }
+      },
+      {
+        "id": "volume-3",
+        "container_path": "/anime",
+        "compose_source_example": "/DATA/AppData/$AppID/Media/Anime",
+        "read_only": false,
+        "required": true,
+        "installation_choice": [
+          "managed-volume",
+          "host-bind"
+        ],
+        "default": "managed-volume",
+        "managed_volume": {
+          "backup": true,
+          "default_size_gb": 8
+        }
       }
     ],
     "ports": [
@@ -182,6 +198,11 @@
               "type": "bind",
               "source": "/DATA/AppData/$AppID/Media/Television",
               "target": "/tv"
+            },
+            {
+              "type": "bind",
+              "source": "/DATA/AppData/$AppID/Media/Anime",
+              "target": "/anime"
             }
           ],
           "network_mode": "bridge",
@@ -234,6 +255,17 @@
         "shared_with_other_lxc": true,
         "source_path": null,
         "source_path_prompt": "Host directory for medusa:/tv"
+      },
+      {
+        "id": "medusa-volume-3",
+        "service": "medusa",
+        "container_path": "/anime",
+        "mode": "host-bind",
+        "user_selectable": true,
+        "backup": false,
+        "shared_with_other_lxc": true,
+        "source_path": null,
+        "source_path_prompt": "Host directory for medusa:/anime"
       }
     ],
     "orchestration": {
@@ -436,6 +468,12 @@
           "remove_lost_found": true,
           "owner_strategy": "mapped-root",
           "only_when_mount_type": "managed-volume"
+        },
+        {
+          "container_path": "/anime",
+          "remove_lost_found": true,
+          "owner_strategy": "mapped-root",
+          "only_when_mount_type": "managed-volume"
         }
       ]
     }

oci/catalog/apps/openspeedtest.json

@@ -304,7 +304,12 @@
         "behavioral_impact": "The application cannot exchange IPC objects with processes on the Proxmox host.",
         "validation": "not-requested-by-compose"
       }
-    ]
+    ],
+    "installer_profile": {
+      "non_persistent_image_volumes": [
+        "/var/log/letsencrypt"
+      ]
+    }
   },
   "compatibility": {
     "automatic_install_candidate": true,

oci/catalog/apps/postgresql.json

@@ -104,7 +104,7 @@
     "volumes": [
       {
         "id": "volume-0",
-        "container_path": "/var/lib/postgresql/data",
+        "container_path": "/var/lib/postgresql",
         "compose_source_example": "/DATA/AppData/postgresql/data",
         "read_only": false,
         "required": true,
@@ -178,7 +178,7 @@
             {
               "type": "bind",
               "source": "/DATA/AppData/postgresql/data",
-              "target": "/var/lib/postgresql/data"
+              "target": "/var/lib/postgresql"
             }
           ],
           "container_name": "postgresql"
@@ -201,7 +201,7 @@
       {
         "id": "postgresql-volume-0",
         "service": "postgresql",
-        "container_path": "/var/lib/postgresql/data",
+        "container_path": "/var/lib/postgresql",
         "mode": "managed-volume",
         "user_selectable": false,
         "backup": true,
@@ -397,7 +397,7 @@
     "installer_profile": {
       "volume_preparations": [
         {
-          "container_path": "/var/lib/postgresql/data",
+          "container_path": "/var/lib/postgresql",
           "remove_lost_found": true,
           "owner_strategy": "mapped-root",
           "only_when_mount_type": "managed-volume"

oci/catalog/apps/threadfin.json

@@ -76,7 +76,7 @@
     "volumes": [
       {
         "id": "volume-0",
-        "container_path": "/home/threadfin/conf/data",
+        "container_path": "/home/threadfin/conf",
         "compose_source_example": "/DATA/AppData/threadfin/config",
         "read_only": false,
         "required": true,
@@ -89,22 +89,6 @@
           "backup": true,
           "default_size_gb": 8
         }
-      },
-      {
-        "id": "volume-1",
-        "container_path": "/home/threadfin/conf/backup",
-        "compose_source_example": "/DATA/AppData/threadfin/backup",
-        "read_only": false,
-        "required": true,
-        "installation_choice": [
-          "managed-volume",
-          "host-bind"
-        ],
-        "default": "managed-volume",
-        "managed_volume": {
-          "backup": true,
-          "default_size_gb": 8
-        }
       }
     ],
     "ports": [
@@ -155,12 +139,7 @@
             {
               "type": "bind",
               "source": "/DATA/AppData/threadfin/config",
-              "target": "/home/threadfin/conf/data"
-            },
-            {
-              "type": "bind",
-              "source": "/DATA/AppData/threadfin/backup",
-              "target": "/home/threadfin/conf/backup"
+              "target": "/home/threadfin/conf"
             }
           ],
           "container_name": "threadfin"
@@ -183,18 +162,7 @@
       {
         "id": "threadfin-volume-0",
         "service": "threadfin",
-        "container_path": "/home/threadfin/conf/data",
-        "mode": "managed-volume",
-        "user_selectable": false,
-        "backup": true,
-        "shared_with_other_lxc": false,
-        "source_path": null,
-        "source_path_prompt": null
-      },
-      {
-        "id": "threadfin-volume-1",
-        "service": "threadfin",
-        "container_path": "/home/threadfin/conf/backup",
+        "container_path": "/home/threadfin/conf",
         "mode": "managed-volume",
         "user_selectable": false,
         "backup": true,
@@ -387,17 +355,14 @@
     "installer_profile": {
       "volume_preparations": [
         {
-          "container_path": "/home/threadfin/conf/data",
-          "remove_lost_found": true,
-          "owner_strategy": "mapped-root",
-          "only_when_mount_type": "managed-volume"
-        },
-        {
-          "container_path": "/home/threadfin/conf/backup",
+          "container_path": "/home/threadfin/conf",
           "remove_lost_found": true,
           "owner_strategy": "mapped-root",
           "only_when_mount_type": "managed-volume"
         }
+      ],
+      "non_persistent_image_volumes": [
+        "/tmp/threadfin"
       ]
     }
   },

oci/catalog/overlays/archivebox.json

@@ -12,6 +12,9 @@
           "owner_strategy": "mapped-root",
           "only_when_mount_type": "managed-volume"
         }
+      ],
+      "non_persistent_image_volumes": [
+        "/out"
       ]
     }
   }

oci/catalog/overlays/cloudflared.json

@@ -7,6 +7,12 @@
           "remove_lost_found": true,
           "owner_strategy": "mapped-root",
           "only_when_mount_type": "managed-volume"
+        },
+        {
+          "container_path": "/root/.cloudflared",
+          "remove_lost_found": true,
+          "owner_strategy": "mapped-root",
+          "only_when_mount_type": "managed-volume"
         }
       ]
     }

oci/catalog/overlays/flaresolverr.json

@@ -0,0 +1,9 @@
+{
+  "proxmox": {
+    "installer_profile": {
+      "non_persistent_image_volumes": [
+        "/config"
+      ]
+    }
+  }
+}

oci/catalog/overlays/handbrake-jlesage.json

@@ -83,6 +83,12 @@
           "remove_lost_found": true,
           "owner_strategy": "mapped-application-user",
           "only_when_mount_type": "managed-volume"
+        },
+        {
+          "container_path": "/trash",
+          "remove_lost_found": true,
+          "owner_strategy": "mapped-application-user",
+          "only_when_mount_type": "managed-volume"
         }
       ],
       "startup_healthcheck": {

oci/catalog/overlays/medusa-official.json

@@ -19,6 +19,12 @@
           "remove_lost_found": true,
           "owner_strategy": "mapped-root",
           "only_when_mount_type": "managed-volume"
+        },
+        {
+          "container_path": "/anime",
+          "remove_lost_found": true,
+          "owner_strategy": "mapped-root",
+          "only_when_mount_type": "managed-volume"
         }
       ]
     }

oci/catalog/overlays/openspeedtest.json

@@ -0,0 +1,9 @@
+{
+  "proxmox": {
+    "installer_profile": {
+      "non_persistent_image_volumes": [
+        "/var/log/letsencrypt"
+      ]
+    }
+  }
+}

oci/catalog/overlays/postgresql.json

@@ -77,7 +77,7 @@
       ],
       "volume_preparations": [
         {
-          "container_path": "/var/lib/postgresql/data",
+          "container_path": "/var/lib/postgresql",
           "remove_lost_found": true,
           "owner_strategy": "mapped-root",
           "only_when_mount_type": "managed-volume"

oci/catalog/overlays/threadfin.json

@@ -3,17 +3,14 @@
     "installer_profile": {
       "volume_preparations": [
         {
-          "container_path": "/home/threadfin/conf/data",
-          "remove_lost_found": true,
-          "owner_strategy": "mapped-root",
-          "only_when_mount_type": "managed-volume"
-        },
-        {
-          "container_path": "/home/threadfin/conf/backup",
+          "container_path": "/home/threadfin/conf",
           "remove_lost_found": true,
           "owner_strategy": "mapped-root",
           "only_when_mount_type": "managed-volume"
         }
+      ],
+      "non_persistent_image_volumes": [
+        "/tmp/threadfin"
       ]
     }
   }

oci/remote/oci_installation_state.py

@@ -211,6 +211,8 @@ def compare(record, current, candidate=None):
         if not managed or options.get('backup') != '1':
             blockers.append('persistent-mount-needs-backup:' + key)
     declared = set(record['image']['defaults'].get('Volumes') or {})
+    declared -= set(record['template'].get('proxmox', {}).get('installer_profile', {})
+                    .get('non_persistent_image_volumes', []))
     declared.update(v['container_path'] for v in record['template'].get('container_contract', {}).get('volumes', []) if v.get('container_path'))
     for path in sorted(declared):
         if path not in mounts:

oci/remote/oci_instance_transaction.py

@@ -57,6 +57,11 @@ SPINNER_FRAME = re.compile('^ ?[' + ''.join(oci_ui.FRAMES) + ']')
 _run = {'log': None, 'pending': [], 'journal': None, 'failed_log': None, 'failed_lines': None}
 
 
+def non_persistent_image_volumes(template):
+    """Image volumes the template states hold no data, which stay in the rootfs."""
+    return set(template.get('proxmox', {}).get('installer_profile', {}).get('non_persistent_image_volumes', []))
+
+
 def screen_text(line):
     """What a terminal shows for one output line, without colours."""
     parts = [SPINNER_FRAME.sub('', ANSI.sub('', part)).rstrip() for part in line.split('\r')]
@@ -488,7 +493,8 @@ def preflight(record, candidate, config, coordinated=None):
         required = {v['container_path'] for v in template['container_contract'].get('volumes', []) if v.get('required', True)}
         if not required <= {m['container_path'] for m in plan.get('mounts', [])}:
             raise ValueError(translate('Required persistent paths cannot be removed'))
-    image_paths = record['observed']['image']['defaults'].get('Volumes') or {}
+    image_paths = set(record['observed']['image']['defaults'].get('Volumes') or {})
+    image_paths -= non_persistent_image_volumes(record['template'])
     if any(not any(p == target or p.startswith(target.rstrip('/') + '/') for target in old) for p in image_paths):
         raise ValueError(translate('The image declares data paths that are still stored in the rootfs'))
     check = effective_healthcheck(candidate['template'])
@@ -885,7 +891,7 @@ def apply(root, vmid, archive, operation, proposal=None, registry_digest=None, i
         return None
     required = {m['container_path'] for m in candidate['deployment'].get('mounts', [])}
     if any(not any(p == target or p.startswith(target.rstrip('/') + '/') for target in required)
-           for p in (image['defaults'].get('Volumes') or {})):
+           for p in set(image['defaults'].get('Volumes') or {}) - non_persistent_image_volumes(candidate['template'])):
         raise ValueError(translate('The new image requires additional persistent paths; use Recreate'))
     directory = instances.location(root, vmid).parent / 'transactions' / uuid.uuid4().hex
     private_directory(directory)

oci/remote/oci_stack_native.py

@@ -269,8 +269,10 @@ class NativeAdapter:
         if image['architecture'] != old['architecture'] or image['os'] != 'linux':
             raise ValueError(translate('Incompatible image platform'))
         paths = [m['container_path'] for m in record['deployment'].get('mounts', [])]
+        transient = set(record.get('template', {}).get('proxmox', {}).get('installer_profile', {})
+                        .get('non_persistent_image_volumes', []))
         if any(not any(p == target or p.startswith(target.rstrip('/') + '/') for target in paths)
-               for p in (image['defaults'].get('Volumes') or {})):
+               for p in set(image['defaults'].get('Volumes') or {}) - transient):
             raise ValueError(translate('The new image requires additional persistent paths'))
         profile = record['deployment'].get('replay_profile', {})
         if profile.get('adapter') in ('install_nextcloud_stack.sh', 'install_paperless_stack.sh', 'install_tandoor_stack.sh', 'install_immich_stack.sh'):

oci/remote/oci_stack_replay.py

@@ -114,7 +114,9 @@ def captured_menu_ready(primary, adapter, convert, expected_roles):
             return False
         for member in converted:
             targets = [m['container_path'] for m in member['deployment']['mounts']]
-            declared = member['observed']['image']['defaults'].get('Volumes') or {}
+            declared = set(member['observed']['image']['defaults'].get('Volumes') or {})
+            declared -= set(member.get('template', {}).get('proxmox', {}).get('installer_profile', {})
+                            .get('non_persistent_image_volumes', []))
             if any(not any(p == target or p.startswith(target.rstrip('/') + '/') for target in targets)
                    for p in declared):
                 return False

oci/src/proxmenux_oci/management.py

@@ -245,7 +245,7 @@ def manage_instance(project, ui, row, action=None, lifecycle_args=()):
                 wizard.close()
             if not approved:
                 return False
-        elif not ui.review(translate('The saved image channel is checked for a newer image. If replacement is needed, the CT is stopped and a native backup is verified before its root is replaced. Host directories are outside that backup.'),
+        elif not ui.review(translate('If the image channel has a new version, the CT is stopped, backed up and verified, and replaced with the new image to update the container. If anything fails, the backup is restored.'),
                            translate('Update OCI'), question=translate('Update now?'), default=True):
             return False
         command = [sys.executable, str(project / 'remote/oci_update_current.py'), str(row['vmid']),

oci/tests/test_image_volume_coverage.py

@@ -0,0 +1,53 @@
+"""Every path an image declares as a volume is mounted, or stated to hold no data,
+so the installed container can be updated."""
+
+from pathlib import Path
+import sys
+import unittest
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "src"))
+sys.path.insert(0, str(ROOT / "remote"))
+
+from proxmenux_oci.catalog import Catalog
+
+# The VOLUME paths of each image, as `skopeo inspect --config` reports them.
+DECLARED = {
+    "postgresql": ["/var/lib/postgresql"],
+    "threadfin": ["/home/threadfin/conf", "/tmp/threadfin"],
+    "handbrake-jlesage": ["/config", "/output", "/storage", "/trash", "/watch"],
+    "medusa-official": ["/anime", "/config", "/downloads", "/tv"],
+    "cloudflared": ["/config", "/root/.cloudflared"],
+    "archivebox": ["/data", "/out"],
+    "flaresolverr": ["/config"],
+    "openspeedtest": ["/var/log/letsencrypt"],
+}
+
+
+class ImageVolumeCoverageTests(unittest.TestCase):
+    catalog = Catalog(ROOT)
+
+    def test_declared_volumes_are_mounted_or_hold_no_data(self):
+        for app, declared in DECLARED.items():
+            template = self.catalog.compose(app)
+            mounts = [v["container_path"] for v in template["container_contract"]["volumes"]]
+            transient = set(template["proxmox"]["installer_profile"].get("non_persistent_image_volumes", []))
+            uncovered = [p for p in declared if p not in transient
+                         and not any(p == m or p.startswith(m.rstrip("/") + "/") for m in mounts)]
+            self.assertEqual(uncovered, [], app)
+
+    def test_postgresql_keeps_the_versioned_data_directory_on_the_volume(self):
+        # PostgreSQL 18 stores PGDATA in /var/lib/postgresql/18/docker.
+        template = self.catalog.compose("postgresql")
+        self.assertEqual([v["container_path"] for v in template["container_contract"]["volumes"]],
+                         ["/var/lib/postgresql"])
+
+    def test_the_updater_leaves_paths_without_data_out(self):
+        import oci_instance_transaction
+        template = {"proxmox": {"installer_profile": {"non_persistent_image_volumes": ["/tmp/threadfin"]}}}
+        self.assertEqual(oci_instance_transaction.non_persistent_image_volumes(template), {"/tmp/threadfin"})
+        self.assertEqual(oci_instance_transaction.non_persistent_image_volumes({}), set())
+
+
+if __name__ == "__main__":
+    unittest.main()