ProxMenux/oci/remote/oci_stack_transaction.py

"""Durable coordination protocol; native PVE adapters are supplied explicitly.

This module does not enable stack updates by itself. The adapter must hold the
instance registry lock, verify guest identities on every call, and implement
idempotent restore/publication using the transaction ID. No individual member
may publish its contract from replace(). All adapter results must be JSON data.
"""
import copy
import fcntl
import json
import os
from pathlib import Path
import stat
import uuid

from oci_installation_state import private_directory
from oci_instances import write
import oci_instance_transaction as member_tx
from oci_ui import translate, msg_info, msg_ok, msg_warn


TERMINAL = {'committed', 'rolled-back'}
PHASES = {'prepared', 'preparing', 'stopping', 'backing-up', 'replacing',
          'starting', 'checking', 'publishing', 'recovering',
          'recovery-failed'} | TERMINAL


def save(path, state, phase):
    if state.get('phase') != phase:
        member_tx.log(f'stack phase: {phase}')
    state['phase'] = phase
    write(path, state)


def member(adapter, vmid):
    describe = getattr(adapter, 'describe', None)
    return describe(vmid) if describe else f'CT {vmid}'


def validate_plan(plan):
    if plan.get('operation') not in ('update', 'recreate'):
        raise ValueError(translate('Invalid stack operation'))
    if plan.get('blockers') or plan.get('missing_members'):
        raise ValueError(translate('The stack needs member adaptations or a verification of missing volumes'))
    members = plan.get('members', [])
    ids = [member['vmid'] for member in members]
    if not ids or any(type(vmid) is not int or vmid < 100 for vmid in ids):
        raise ValueError(translate('Invalid stack members'))
    if len(set(ids)) != len(ids) or plan.get('primary_vmid') not in ids:
        raise ValueError(translate('Invalid main member or duplicated members'))
    for key in ('start_order', 'stop_order'):
        order = plan.get(key, [])
        if len(order) != len(ids) or set(order) != set(ids):
            raise ValueError(translate('Incomplete stack order'))
    if plan['start_order'][-1] != plan['primary_vmid'] or plan['stop_order'][0] != plan['primary_vmid']:
        raise ValueError(translate('The main member must stop first and start last'))


def recover_state(path, state, adapter):
    """Recover every private backup once replacement could have begun.

    A dependency may receive database writes even when only the frontend was
    replaced. Consequently rollback never restores just the failing member.
    External host files are deliberately outside this recovery protocol.
    """
    if state['phase'] in TERMINAL:
        if hasattr(adapter, 'finalize'):
            adapter.finalize(state)
        return state
    ids = {str(vmid) for vmid in state['plan']['start_order']}
    if (set(state.get('running', {})) != ids
            or any(type(value) is not bool for value in state['running'].values())
            or type(state.get('stop_intent')) is not bool
            or type(state.get('replacement_intent')) is not bool):
        raise ValueError(translate('The journal has an incomplete recovery state'))
    if state['replacement_intent'] and (not state['stop_intent'] or set(state.get('backups', {})) != ids):
        raise ValueError(translate('Stack backups are missing; a partial restore is not allowed'))
    adapter.validate(state['plan'])
    if state['replacement_intent'] and hasattr(adapter, 'verify_backups'):
        msg_info(translate('Verifying the backups...'))
        adapter.verify_backups(state['backups'])
        msg_ok(translate('Backups verified'))
    save(path, state, 'recovering')
    try:
        if state['stop_intent']:
            msg_info(translate('Stopping the stack...'))
            for vmid in state['plan']['stop_order']:
                adapter.stop(vmid)
            msg_ok(translate('Stack stopped'))
        if state['replacement_intent']:
            for vmid in state['plan']['start_order']:
                backup = state['backups'].get(str(vmid))
                if backup is None:
                    raise ValueError(translate('A stack backup is missing; a partial restore is not allowed'))
            for vmid in state['plan']['start_order']:
                msg_info(f"{translate('Restoring')} {member(adapter, vmid)}...")
                adapter.restore(vmid, state['backups'][str(vmid)], state['id'])
                msg_ok(f"{translate('Restored:')} {member(adapter, vmid)}")
            msg_info(translate('Restoring the stack records...'))
            adapter.restore_contracts(state['plan'], state['id'])
            msg_ok(translate('Stack records restored'))
        if state['stop_intent']:
            msg_info(translate('Returning the containers to their previous state...'))
            adapter.restore_running_state(state['running'], state['plan']['start_order'])
            msg_ok(translate('Containers returned to their previous state'))
        save(path, state, 'rolled-back')
    except Exception:
        save(path, state, 'recovery-failed')
        raise
    if hasattr(adapter, 'finalize'):
        adapter.finalize(state)
    return state


def _apply(path, plan, adapter):
    validate_plan(plan)
    if path.exists():
        raise ValueError(translate('A journal already exists; review or recover it before trying again'))
    adapter.validate(plan)
    running = {str(vmid): adapter.is_running(vmid) for vmid in plan['start_order']}
    if any(type(value) is not bool for value in running.values()):
        raise ValueError(translate('Invalid running state'))
    state = {'schema_version': 1, 'id': str(uuid.uuid4()),
             'plan': copy.deepcopy(plan), 'running': running,
             'prepared': {}, 'backups': {}, 'stop_intent': False,
             'replacement_intent': False}
    save(path, state, 'prepared')
    try:
        # Resolve/download/verify every candidate before stopping any service.
        save(path, state, 'preparing')
        for candidate in plan['members']:
            state['prepared'][str(candidate['vmid'])] = adapter.prepare(candidate, plan['operation'])
            save(path, state, 'preparing')
        adapter.validate(plan)
        state['stop_intent'] = True
        save(path, state, 'stopping')
        msg_info(translate('Stopping the stack...'))
        for vmid in plan['stop_order']:
            adapter.stop(vmid)
        msg_ok(translate('Stack stopped'))
        save(path, state, 'backing-up')
        for vmid in plan['start_order']:
            msg_info(f"{translate('Creating a backup of')} {member(adapter, vmid)}...")
            state['backups'][str(vmid)] = adapter.backup(vmid, state['id'])
            save(path, state, 'backing-up')
            msg_ok(f"{translate('Backup created:')} {member(adapter, vmid)}")
        # The adapter must verify all archives, free space and device identities.
        msg_info(translate('Verifying the backups...'))
        adapter.verify_backups(state['backups'])
        adapter.validate(plan)
        msg_ok(translate('Backups verified'))
        state['replacement_intent'] = True
        save(path, state, 'replacing')
        for vmid in plan['start_order']:
            msg_info(f"{translate('Updating')} {member(adapter, vmid)}...")
            adapter.replace(vmid, state['prepared'][str(vmid)], state['id'])
            msg_ok(f"{translate('Updated:')} {member(adapter, vmid)}")
        save(path, state, 'starting')
        for vmid in plan['start_order']:
            msg_info(f"{translate('Starting')} {member(adapter, vmid)}...")
            adapter.start(vmid)
            adapter.healthcheck(vmid)
            msg_ok(f"{translate('Service responding:')} {member(adapter, vmid)}")
        save(path, state, 'checking')
        msg_info(translate('Checking the updated stack...'))
        adapter.validate_candidates(state)
        adapter.restore_running_state(running, plan['start_order'])
        msg_ok(translate('Updated stack checked'))
        save(path, state, 'publishing')
        msg_info(translate('Saving the stack records...'))
        adapter.publish(state)
        save(path, state, 'committed')
        msg_ok(translate('Stack records saved'))
    except Exception as error:
        member_tx.report_error(error)
        try:
            error.oci_reported = True
        except AttributeError:
            pass
        if state['stop_intent']:
            msg_warn(translate('Restoring the previous state of the stack...'))
        recover_state(path, state, adapter)
        raise
    if hasattr(adapter, 'finalize'):
        adapter.finalize(state)
    return state


def execute(journal, adapter, plan=None):
    """Apply when plan is supplied; otherwise explicitly recover an old journal."""
    path = Path(journal)
    private_directory(path.parent)
    lock = path.with_name(path.name + '.lock')
    if path.is_symlink() or lock.is_symlink():
        raise ValueError(translate('Unsafe journal or lock file'))
    with lock.open('a') as handle:
        lock.chmod(0o600)
        fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
        if plan is not None:
            return _apply(path, plan, adapter)
        attributes = path.lstat()
        if (not stat.S_ISREG(attributes.st_mode) or attributes.st_uid != os.geteuid()
                or attributes.st_mode & 0o077):
            raise ValueError(translate('The private journal has an unsafe owner or permissions'))
        state = json.loads(path.read_text())
        if state.get('schema_version') != 1 or state.get('phase') not in PHASES:
            raise ValueError(translate('Invalid stack journal'))
        validate_plan(state['plan'])
        return recover_state(path, state, adapter)