mirror of
https://github.com/Akkudoktor-EOS/EOS.git
synced 2026-09-14 03:46:38 +00:00
fix: handle denied inspection in test server cleanup (#1301)
Track owned test processes and restrict fallback cleanup to verified EOS servers using the test configuration. Add regression coverage for denied inspection and cleanup failures. Fixes #1297
This commit is contained in:
dr-dimitri
+125
-118
@@ -5,16 +5,16 @@ import json
|
||||
import logging
|
||||
import os
|
||||
import pickle
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from collections.abc import Sequence
|
||||
from contextlib import contextmanager
|
||||
from fnmatch import fnmatch
|
||||
from http import HTTPStatus
|
||||
from pathlib import Path
|
||||
from typing import Callable, Generator, Optional, Union, cast
|
||||
from typing import Callable, Generator, Optional, TextIO, Union, cast
|
||||
from unittest.mock import PropertyMock, patch
|
||||
|
||||
import pandas as pd
|
||||
@@ -409,127 +409,121 @@ def config_eos(config_eos_factory) -> ConfigEOS:
|
||||
# ------------------------------------
|
||||
|
||||
|
||||
def _test_server_process(pid: int, module: str, config_dir: str) -> Optional[psutil.Process]:
|
||||
"""Verify that a fallback PID belongs to this test's EOS configuration."""
|
||||
if pid <= 0 or pid == os.getpid():
|
||||
return None
|
||||
try:
|
||||
process = psutil.Process(pid)
|
||||
cmdline = process.cmdline()
|
||||
script = Path(__file__).parent.parent / "src" / Path(*module.split("."))
|
||||
is_module = cmdline[1:3] == ["-m", module]
|
||||
is_script = len(cmdline) > 1 and Path(cmdline[1]).resolve() == script.with_suffix(".py")
|
||||
if not (is_module or is_script):
|
||||
return None
|
||||
process_config_dir = process.environ().get("EOS_CONFIG_DIR")
|
||||
if process_config_dir and Path(process_config_dir).resolve() == Path(config_dir).resolve():
|
||||
return process
|
||||
except (psutil.Error, OSError):
|
||||
# Protected or exited processes cannot be verified and must be left alone.
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def cleanup_eos_eosdash(
|
||||
host: str,
|
||||
port: int,
|
||||
eosdash_host: str,
|
||||
eosdash_port: int,
|
||||
server_timeout: float = 10.0,
|
||||
*,
|
||||
owned_processes: Sequence[psutil.Process] = (),
|
||||
config_dir: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Clean up any running EOS and EOSdash processes.
|
||||
"""Stop owned test processes and verified servers using the test configuration.
|
||||
|
||||
Process objects retain process identity across PID reuse. Health endpoints and
|
||||
connection inspection are only fallbacks for restarted or orphaned servers;
|
||||
neither a port match nor a reported PID alone authorizes termination.
|
||||
|
||||
Args:
|
||||
host (str): EOS server host (e.g., "127.0.0.1").
|
||||
port (int): Port number used by the EOS process.
|
||||
eosdash_hostr (str): EOSdash server host.
|
||||
eosdash_port (int): Port used by EOSdash.
|
||||
server_timeout (float): Timeout in seconds before giving up.
|
||||
host: EOS server host.
|
||||
port: EOS server port.
|
||||
eosdash_host: EOSdash server host.
|
||||
eosdash_port: EOSdash server port.
|
||||
server_timeout: Maximum time allowed for HTTP probes and termination waits.
|
||||
owned_processes: Process handles captured by the test that started them.
|
||||
config_dir: Unique test configuration directory required for fallback cleanup.
|
||||
"""
|
||||
server = f"http://{host}:{port}"
|
||||
eosdash_server = f"http://{eosdash_host}:{eosdash_port}"
|
||||
deadline = time.monotonic() + server_timeout
|
||||
processes = list(owned_processes)
|
||||
servers = (
|
||||
(f"http://{host}:{port}/v1/health", port, "akkudoktoreos.server.eos"),
|
||||
(
|
||||
f"http://{eosdash_host}:{eosdash_port}/eosdash/health",
|
||||
eosdash_port,
|
||||
"akkudoktoreos.server.eosdash",
|
||||
),
|
||||
)
|
||||
|
||||
sigkill = signal.SIGTERM if os.name == "nt" else signal.SIGKILL
|
||||
|
||||
# Attempt to shut down EOS via health endpoint
|
||||
if config_dir is not None:
|
||||
for url, _, module in servers:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
break
|
||||
try:
|
||||
result = requests.get(f"{server}/v1/health", timeout=2)
|
||||
if result.status_code == HTTPStatus.OK:
|
||||
pid = result.json()["pid"]
|
||||
os.kill(pid, sigkill)
|
||||
time.sleep(1)
|
||||
result = requests.get(f"{server}/v1/health", timeout=2)
|
||||
assert result.status_code != HTTPStatus.OK
|
||||
except Exception:
|
||||
response = requests.get(url, timeout=min(2.0, remaining))
|
||||
if response.status_code == HTTPStatus.OK:
|
||||
pid = response.json()["pid"]
|
||||
if type(pid) is int:
|
||||
process = _test_server_process(pid, module, config_dir)
|
||||
if process is not None:
|
||||
processes.append(process)
|
||||
except (requests.RequestException, ValueError, KeyError, TypeError):
|
||||
pass
|
||||
|
||||
# Fallback: kill processes bound to the EOS port
|
||||
pids: list[int] = []
|
||||
for _ in range(int(server_timeout / 3)):
|
||||
for conn in psutil.net_connections(kind="inet"):
|
||||
if conn.laddr and conn.laddr.port == port and conn.pid is not None:
|
||||
# macOS may deny the entire enumeration because of an unrelated process.
|
||||
# Inspect once; owned handles and verified health PIDs work without it.
|
||||
try:
|
||||
process = psutil.Process(conn.pid)
|
||||
cmdline = process.as_dict(attrs=["cmdline"])["cmdline"]
|
||||
if "akkudoktoreos.server.eos" in " ".join(cmdline):
|
||||
pids.append(conn.pid)
|
||||
except Exception:
|
||||
connections = psutil.net_connections(kind="inet")
|
||||
except (psutil.AccessDenied, OSError):
|
||||
connections = []
|
||||
for conn in connections:
|
||||
if not conn.laddr or conn.pid is None:
|
||||
continue
|
||||
for _, server_port, module in servers:
|
||||
if conn.laddr.port == server_port:
|
||||
process = _test_server_process(conn.pid, module, config_dir)
|
||||
if process is not None:
|
||||
processes.append(process)
|
||||
|
||||
# Capture descendants before stopping parents, which may otherwise orphan them.
|
||||
roots = list(dict.fromkeys(processes))
|
||||
for process in roots:
|
||||
try:
|
||||
processes.extend(process.children(recursive=True))
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied):
|
||||
pass
|
||||
for pid in pids:
|
||||
os.kill(pid, sigkill)
|
||||
running = False
|
||||
for pid in pids:
|
||||
processes = list(dict.fromkeys(processes))
|
||||
for process in processes:
|
||||
try:
|
||||
proc = psutil.Process(pid)
|
||||
status = proc.status()
|
||||
if status != psutil.STATUS_ZOMBIE:
|
||||
running = True
|
||||
break
|
||||
# Stop supervisors first so they cannot respawn their children.
|
||||
if os.name == "nt":
|
||||
process.terminate()
|
||||
else:
|
||||
process.kill()
|
||||
except psutil.NoSuchProcess:
|
||||
continue
|
||||
if not running:
|
||||
break
|
||||
time.sleep(3)
|
||||
|
||||
# Check for processes still running (maybe zombies).
|
||||
for pid in pids:
|
||||
try:
|
||||
proc = psutil.Process(pid)
|
||||
status = proc.status()
|
||||
assert status == psutil.STATUS_ZOMBIE, f"Cleanup EOS expected zombie, got {status} for PID {pid}"
|
||||
except psutil.NoSuchProcess:
|
||||
# Process already reaped (possibly by init/systemd)
|
||||
continue
|
||||
|
||||
# Attempt to shut down EOSdash via health endpoint
|
||||
for srv in (eosdash_server, "http://127.0.0.1:8504", "http://127.0.0.1:8555"):
|
||||
try:
|
||||
result = requests.get(f"{srv}/eosdash/health", timeout=2)
|
||||
if result.status_code == HTTPStatus.OK:
|
||||
pid = result.json()["pid"]
|
||||
os.kill(pid, sigkill)
|
||||
time.sleep(1)
|
||||
result = requests.get(f"{srv}/eosdash/health", timeout=2)
|
||||
assert result.status_code != HTTPStatus.OK
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fallback: kill EOSdash processes bound to known ports
|
||||
pids = []
|
||||
for _ in range(int(server_timeout / 3)):
|
||||
for conn in psutil.net_connections(kind="inet"):
|
||||
if conn.laddr and conn.laddr.port in (eosdash_port, 8504, 8555) and conn.pid is not None:
|
||||
_, alive = psutil.wait_procs(processes, timeout=max(0.0, deadline - time.monotonic()))
|
||||
running = []
|
||||
for process in alive:
|
||||
try:
|
||||
process = psutil.Process(conn.pid)
|
||||
cmdline = process.as_dict(attrs=["cmdline"])["cmdline"]
|
||||
if "akkudoktoreos.server.eosdash" in " ".join(cmdline):
|
||||
pids.append(conn.pid)
|
||||
except Exception:
|
||||
if process.is_running() and process.status() != psutil.STATUS_ZOMBIE:
|
||||
running.append(process.pid)
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
for pid in pids:
|
||||
os.kill(pid, sigkill)
|
||||
running = False
|
||||
for pid in pids:
|
||||
try:
|
||||
proc = psutil.Process(pid)
|
||||
status = proc.status()
|
||||
if status != psutil.STATUS_ZOMBIE:
|
||||
running = True
|
||||
break
|
||||
except psutil.NoSuchProcess:
|
||||
continue
|
||||
if not running:
|
||||
break
|
||||
time.sleep(3)
|
||||
|
||||
# Check for processes still running (maybe zombies).
|
||||
for pid in pids:
|
||||
try:
|
||||
proc = psutil.Process(pid)
|
||||
status = proc.status()
|
||||
assert status == psutil.STATUS_ZOMBIE, f"Cleanup EOSdash expected zombie, got {status} for PID {pid}"
|
||||
except psutil.NoSuchProcess:
|
||||
# Process already reaped (possibly by init/systemd)
|
||||
continue
|
||||
assert not running, f"Test server cleanup timed out for PIDs {running}"
|
||||
|
||||
|
||||
@contextmanager
|
||||
@@ -573,6 +567,8 @@ def server_base(
|
||||
eos_tmp_dir = tempfile.TemporaryDirectory()
|
||||
eos_dir = str(eos_tmp_dir.name)
|
||||
eos_general_data_folder_path = str(Path(eos_dir) / "data")
|
||||
process_name = f"eos-{Path(eos_dir).name}"
|
||||
owned_processes: list[psutil.Process] = []
|
||||
|
||||
class Starter(ProcessStarter):
|
||||
# Set environment for server run
|
||||
@@ -636,12 +632,21 @@ def server_base(
|
||||
# xprocess will now attempt to clean up upon interruptions
|
||||
terminate_on_interrupt = True
|
||||
|
||||
def wait(self, log_file: TextIO) -> bool:
|
||||
"""Capture the process identity even if the startup check fails."""
|
||||
pid = self.process.getinfo(process_name).pid
|
||||
owned_processes.append(psutil.Process(pid))
|
||||
return super().wait(log_file)
|
||||
|
||||
# checks if our server is ready
|
||||
def startup_check(self):
|
||||
try:
|
||||
response = requests.get(f"{server}/v1/health", timeout=10)
|
||||
logger.debug(f"[xprocess] Health check: {response.status_code}")
|
||||
if response.status_code == 200:
|
||||
if (
|
||||
response.status_code == 200
|
||||
and response.json().get("pid") == owned_processes[0].pid
|
||||
):
|
||||
return True
|
||||
logger.debug(f"[xprocess] Health check: {response}")
|
||||
except Exception as e:
|
||||
@@ -659,7 +664,7 @@ def server_base(
|
||||
if self.startup_check():
|
||||
return True
|
||||
if datetime.now() > self._max_time:
|
||||
info = self.process.getinfo("eos")
|
||||
info = self.process.getinfo(process_name)
|
||||
error_msg = (
|
||||
f"The provided startup check could not assert process responsiveness\n"
|
||||
f"within the specified time interval of {self.timeout} seconds.\n"
|
||||
@@ -667,19 +672,17 @@ def server_base(
|
||||
)
|
||||
raise TimeoutError(error_msg)
|
||||
|
||||
# Kill all running eos and eosdash process - just to be sure
|
||||
cleanup_eos_eosdash(host, port, eosdash_host, eosdash_port, server_timeout)
|
||||
|
||||
# Ensure there is an empty config file in the temporary EOS directory
|
||||
config_file_path = Path(eos_dir).joinpath(ConfigEOS.CONFIG_FILE_NAME)
|
||||
with config_file_path.open(mode="w", encoding="utf-8", newline="\n") as fd:
|
||||
json.dump({}, fd)
|
||||
logger.info(f"Created empty config file in {config_file_path}.")
|
||||
|
||||
# ensure process is running and return its logfile
|
||||
pid, logfile = xprocess.ensure("eos", Starter)
|
||||
logger.info(f"Started EOS ({pid}). This may take very long (up to {server_timeout} seconds).")
|
||||
logger.info(f"EOS_DIR: {Starter.env["EOS_DIR"]}, EOS_CONFIG_DIR: {Starter.env["EOS_CONFIG_DIR"]}")
|
||||
try:
|
||||
# A unique name prevents xprocess from reusing a different test's server.
|
||||
pid, logfile = xprocess.ensure(process_name, Starter)
|
||||
logger.info(f"Started EOS ({pid}). This may take up to {server_timeout} seconds.")
|
||||
logger.info(f"EOS_DIR: {eos_dir}, EOS_CONFIG_DIR: {eos_dir}")
|
||||
logger.info(f"View xprocess logfile at: {logfile}")
|
||||
|
||||
yield {
|
||||
@@ -690,14 +693,18 @@ def server_base(
|
||||
"eos_dir": eos_dir,
|
||||
"timeout": server_timeout,
|
||||
}
|
||||
|
||||
# clean up whole process tree afterwards
|
||||
xprocess.getinfo("eos").terminate()
|
||||
|
||||
# Cleanup any EOS process left.
|
||||
cleanup_eos_eosdash(host, port, eosdash_host, eosdash_port, server_timeout)
|
||||
|
||||
# Remove temporary EOS_DIR
|
||||
finally:
|
||||
try:
|
||||
cleanup_eos_eosdash(
|
||||
host,
|
||||
port,
|
||||
eosdash_host,
|
||||
eosdash_port,
|
||||
server_timeout,
|
||||
owned_processes=owned_processes,
|
||||
config_dir=eos_dir,
|
||||
)
|
||||
finally:
|
||||
eos_tmp_dir.cleanup()
|
||||
|
||||
|
||||
|
||||
+22
-46
@@ -1,7 +1,6 @@
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import time
|
||||
from http import HTTPStatus
|
||||
from pathlib import Path
|
||||
@@ -170,7 +169,7 @@ class TestServerStartStop:
|
||||
monkeypatch.setenv("EOS_CONFIG_DIR", str(eos_dir))
|
||||
|
||||
# Import with environment vars set to prevent creation of EOS.config.json in wrong dir.
|
||||
from akkudoktoreos.server.rest.starteosdash import supervise_eosdash
|
||||
from akkudoktoreos.server.rest import starteosdash
|
||||
|
||||
config_eos.server.host = get_default_host()
|
||||
config_eos.server.port = 8503
|
||||
@@ -180,79 +179,56 @@ class TestServerStartStop:
|
||||
|
||||
eosdash_server = f"http://{config_eos.server.eosdash_host}:{config_eos.server.eosdash_port}"
|
||||
|
||||
# Cleanup any EOS and EOSdash process left.
|
||||
cleanup_eos_eosdash(
|
||||
host=config_eos.server.host,
|
||||
port=config_eos.server.port,
|
||||
eosdash_host=config_eos.server.eosdash_host,
|
||||
eosdash_port=config_eos.server.eosdash_port,
|
||||
server_timeout=timeout,
|
||||
)
|
||||
|
||||
# Port may be blocked
|
||||
assert wait_for_port_free(config_eos.server.eosdash_port, timeout=120, waiting_app_name="EOSdash")
|
||||
|
||||
"""Start EOSdash."""
|
||||
await supervise_eosdash()
|
||||
owned_processes: list[psutil.Process] = []
|
||||
try:
|
||||
await starteosdash.supervise_eosdash()
|
||||
process = starteosdash.eosdash_proc
|
||||
assert process is not None, "EOSdash supervisor did not start a process"
|
||||
owned_processes.append(psutil.Process(process.pid))
|
||||
|
||||
# give EOSdash some time to startup
|
||||
await asyncio.sleep(1)
|
||||
|
||||
# ---------------------------------
|
||||
# Wait for health endpoint to come up
|
||||
# ---------------------------------
|
||||
startup = False
|
||||
error = ""
|
||||
|
||||
for retries in range(int(timeout / 3)):
|
||||
for _ in range(int(timeout / 3)):
|
||||
try:
|
||||
resp = requests.get(f"{eosdash_server}/eosdash/health", timeout=2)
|
||||
if resp.status_code == HTTPStatus.OK:
|
||||
startup = True
|
||||
break
|
||||
error = f"{resp.status_code}, {str(resp.content)}"
|
||||
except Exception as ex:
|
||||
except requests.RequestException as ex:
|
||||
error = str(ex)
|
||||
|
||||
await asyncio.sleep(3)
|
||||
|
||||
assert startup, f"Connection to {eosdash_server}/eosdash/health failed: {error}"
|
||||
|
||||
health = resp.json()
|
||||
assert health.get("status") == "alive"
|
||||
assert health.get("version") == __version__
|
||||
assert health.get("pid") == process.pid
|
||||
|
||||
# ---------------------------------
|
||||
# Shutdown EOSdash (as provided)
|
||||
# ---------------------------------
|
||||
# Terminate the process started by this test, then reap it via asyncio.
|
||||
process.terminate()
|
||||
await asyncio.wait_for(process.wait(), timeout=timeout)
|
||||
try:
|
||||
resp = requests.get(f"{eosdash_server}/eosdash/health", timeout=2)
|
||||
if resp.status_code == HTTPStatus.OK:
|
||||
pid = resp.json().get("pid")
|
||||
assert pid is not None, "EOSdash did not report a PID"
|
||||
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
time.sleep(1)
|
||||
|
||||
# After shutdown, the server should not respond OK anymore
|
||||
try:
|
||||
resp2 = requests.get(f"{eosdash_server}/eosdash/health", timeout=2)
|
||||
assert resp2.status_code != HTTPStatus.OK
|
||||
except Exception:
|
||||
pass # expected
|
||||
except Exception:
|
||||
pass # ignore shutdown errors for safety
|
||||
|
||||
# ---------------------------------
|
||||
# Cleanup any leftover processes
|
||||
# ---------------------------------
|
||||
except requests.RequestException:
|
||||
pass
|
||||
else:
|
||||
assert resp.status_code != HTTPStatus.OK
|
||||
finally:
|
||||
cleanup_eos_eosdash(
|
||||
host=config_eos.server.host,
|
||||
port=config_eos.server.port,
|
||||
eosdash_host=config_eos.server.eosdash_host,
|
||||
eosdash_port=config_eos.server.eosdash_port,
|
||||
server_timeout=timeout,
|
||||
owned_processes=owned_processes,
|
||||
config_dir=str(config_eos.general.config_folder_path),
|
||||
)
|
||||
if starteosdash.eosdash_proc is not None:
|
||||
await asyncio.wait_for(starteosdash.eosdash_proc.wait(), timeout=timeout)
|
||||
|
||||
@pytest.mark.skipif(os.name == "nt", reason="Server restart not supported on Windows")
|
||||
def test_server_restart(self, server_setup_for_function, is_system_test):
|
||||
|
||||
@@ -0,0 +1,305 @@
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import psutil
|
||||
import pytest
|
||||
import requests
|
||||
from conftest import cleanup_eos_eosdash, server_base
|
||||
from xprocess import ProcessStarter
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cleanup_environment(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> SimpleNamespace:
|
||||
"""Isolate every process and network operation performed by server cleanup."""
|
||||
processes: dict[int, Mock] = {}
|
||||
process_type = psutil.Process
|
||||
|
||||
def make_process(pid: int, module: str = "akkudoktoreos.server.eos") -> Mock:
|
||||
process = Mock(spec=process_type)
|
||||
# Track either platform's termination method with the same mock.
|
||||
process.terminate = process.kill
|
||||
process.pid = pid
|
||||
process.cmdline.return_value = ["python", "-m", module]
|
||||
process.environ.return_value = {"EOS_CONFIG_DIR": str(tmp_path)}
|
||||
process.children.return_value = []
|
||||
process.status.return_value = psutil.STATUS_RUNNING
|
||||
processes[pid] = process
|
||||
return process
|
||||
|
||||
def get_process(pid: int) -> Mock:
|
||||
if pid not in processes:
|
||||
raise psutil.NoSuchProcess(pid)
|
||||
return processes[pid]
|
||||
|
||||
connections = Mock(return_value=[])
|
||||
health = Mock(side_effect=requests.ConnectionError)
|
||||
wait = Mock(return_value=([], []))
|
||||
monkeypatch.setattr("conftest.psutil.Process", get_process)
|
||||
monkeypatch.setattr("conftest.psutil.net_connections", connections)
|
||||
monkeypatch.setattr("conftest.psutil.wait_procs", wait)
|
||||
monkeypatch.setattr("conftest.requests.get", health)
|
||||
|
||||
return SimpleNamespace(
|
||||
make_process=make_process,
|
||||
connections=connections,
|
||||
health=health,
|
||||
wait=wait,
|
||||
config_dir=str(tmp_path),
|
||||
)
|
||||
|
||||
|
||||
def run_cleanup(environment: SimpleNamespace, *processes: psutil.Process) -> None:
|
||||
cleanup_eos_eosdash(
|
||||
"127.0.0.1",
|
||||
8503,
|
||||
"127.0.0.1",
|
||||
8555,
|
||||
owned_processes=processes,
|
||||
config_dir=environment.config_dir,
|
||||
)
|
||||
|
||||
|
||||
def connection(pid: int | None, port: int) -> SimpleNamespace:
|
||||
return SimpleNamespace(pid=pid, laddr=SimpleNamespace(port=port))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("error", [psutil.AccessDenied(1), PermissionError(1, "Denied")])
|
||||
def test_cleanup_owned_tree_when_enumeration_is_denied(
|
||||
cleanup_environment: SimpleNamespace, error: Exception
|
||||
) -> None:
|
||||
"""Protected system processes must not prevent termination of owned servers."""
|
||||
env = cleanup_environment
|
||||
parent = env.make_process(101)
|
||||
child = env.make_process(102, "akkudoktoreos.server.eosdash")
|
||||
parent.children.return_value = [child]
|
||||
env.connections.side_effect = error
|
||||
|
||||
run_cleanup(env, parent)
|
||||
|
||||
parent.kill.assert_called_once_with()
|
||||
child.kill.assert_called_once_with()
|
||||
env.connections.assert_called_once_with(kind="inet")
|
||||
assert env.wait.call_args.args[0] == [parent, child]
|
||||
assert 0 <= env.wait.call_args.kwargs["timeout"] <= 10
|
||||
|
||||
|
||||
def test_cleanup_uses_verified_health_pid_without_connection_inspection(
|
||||
cleanup_environment: SimpleNamespace,
|
||||
) -> None:
|
||||
"""A restarted test server can be found without system-wide connections."""
|
||||
env = cleanup_environment
|
||||
restarted = env.make_process(103)
|
||||
# EOS restarts using the script path rather than `python -m`.
|
||||
script = Path(__file__).parent.parent / "src/akkudoktoreos/server/eos.py"
|
||||
restarted.cmdline.return_value = ["python", str(script)]
|
||||
response = Mock(status_code=200)
|
||||
response.json.return_value = {"pid": restarted.pid}
|
||||
env.health.side_effect = [response, requests.ConnectionError()]
|
||||
env.connections.side_effect = psutil.AccessDenied(1)
|
||||
|
||||
run_cleanup(env)
|
||||
|
||||
restarted.kill.assert_called_once_with()
|
||||
|
||||
|
||||
def test_cleanup_connection_fallback_is_limited_to_verified_test_servers(
|
||||
cleanup_environment: SimpleNamespace,
|
||||
) -> None:
|
||||
"""Port matches alone must not kill other applications or another EOS instance."""
|
||||
env = cleanup_environment
|
||||
eos = env.make_process(101)
|
||||
dashboard = env.make_process(102, "akkudoktoreos.server.eosdash")
|
||||
other_application = env.make_process(103, "unrelated.application")
|
||||
other_test = env.make_process(104)
|
||||
other_test.environ.return_value = {"EOS_CONFIG_DIR": "/another/test"}
|
||||
misleading_module = env.make_process(105, "akkudoktoreos.server.eos_extra")
|
||||
wrong_port = env.make_process(106, "akkudoktoreos.server.eosdash")
|
||||
env.connections.return_value = [
|
||||
connection(101, 8503),
|
||||
connection(101, 8503),
|
||||
connection(102, 8555),
|
||||
connection(103, 8503),
|
||||
connection(104, 8503),
|
||||
connection(105, 8503),
|
||||
connection(106, 8504),
|
||||
connection(None, 8503),
|
||||
]
|
||||
|
||||
run_cleanup(env)
|
||||
|
||||
eos.kill.assert_called_once_with()
|
||||
dashboard.kill.assert_called_once_with()
|
||||
for process in (other_application, other_test, misleading_module, wrong_port):
|
||||
process.kill.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pid", [103, True, "103", -1, None])
|
||||
def test_cleanup_does_not_trust_health_pid(
|
||||
cleanup_environment: SimpleNamespace, pid: object
|
||||
) -> None:
|
||||
"""A health response cannot authorize termination of an unrelated process."""
|
||||
env = cleanup_environment
|
||||
unrelated = env.make_process(103, "unrelated.application")
|
||||
response = Mock(status_code=200)
|
||||
response.json.return_value = {"pid": pid}
|
||||
env.health.side_effect = [response, requests.ConnectionError()]
|
||||
|
||||
run_cleanup(env)
|
||||
|
||||
unrelated.kill.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("operation", ["cmdline", "environ"])
|
||||
def test_cleanup_skips_inaccessible_fallback_process(
|
||||
cleanup_environment: SimpleNamespace, operation: str
|
||||
) -> None:
|
||||
"""One inaccessible process must not hide a subsequent verified server."""
|
||||
env = cleanup_environment
|
||||
protected = env.make_process(103)
|
||||
getattr(protected, operation).side_effect = psutil.AccessDenied(103)
|
||||
owned = env.make_process(104)
|
||||
env.connections.return_value = [connection(103, 8503), connection(104, 8503)]
|
||||
|
||||
run_cleanup(env)
|
||||
|
||||
protected.kill.assert_not_called()
|
||||
owned.kill.assert_called_once_with()
|
||||
|
||||
|
||||
def test_cleanup_stops_owned_process_when_children_are_inaccessible(
|
||||
cleanup_environment: SimpleNamespace,
|
||||
) -> None:
|
||||
env = cleanup_environment
|
||||
owned = env.make_process(101)
|
||||
owned.children.side_effect = psutil.AccessDenied(101)
|
||||
|
||||
run_cleanup(env, owned)
|
||||
|
||||
owned.kill.assert_called_once_with()
|
||||
|
||||
|
||||
def test_cleanup_tolerates_process_exit_during_termination(
|
||||
cleanup_environment: SimpleNamespace,
|
||||
) -> None:
|
||||
env = cleanup_environment
|
||||
exited = env.make_process(101)
|
||||
exited.kill.side_effect = psutil.NoSuchProcess(101)
|
||||
remaining = env.make_process(102)
|
||||
|
||||
run_cleanup(env, exited, remaining)
|
||||
|
||||
remaining.kill.assert_called_once_with()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", [psutil.STATUS_ZOMBIE, psutil.STATUS_RUNNING])
|
||||
def test_cleanup_reports_only_live_processes_after_bounded_wait(
|
||||
cleanup_environment: SimpleNamespace, status: str
|
||||
) -> None:
|
||||
env = cleanup_environment
|
||||
process = env.make_process(101)
|
||||
process.status.return_value = status
|
||||
env.wait.return_value = ([], [process])
|
||||
|
||||
if status == psutil.STATUS_RUNNING:
|
||||
with pytest.raises(AssertionError, match="cleanup timed out.*101"):
|
||||
run_cleanup(env, process)
|
||||
else:
|
||||
run_cleanup(env, process)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("startup_failure", [False, True])
|
||||
def test_server_base_cleans_owned_process_and_directory_on_failure(
|
||||
cleanup_environment: SimpleNamespace,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
startup_failure: bool,
|
||||
) -> None:
|
||||
"""Both startup failures and exceptions from the test body run teardown."""
|
||||
env = cleanup_environment
|
||||
process = env.make_process(101)
|
||||
xprocess = Mock()
|
||||
xprocess.getinfo.return_value.pid = process.pid
|
||||
config_dirs: list[Path] = []
|
||||
monkeypatch.setattr("conftest.subprocess.run", Mock())
|
||||
monkeypatch.setattr("conftest.ProcessStarter.wait", Mock(return_value=True))
|
||||
|
||||
def ensure(name: str, starter_type: type[ProcessStarter]) -> tuple[int, str]:
|
||||
config_dirs.append(Path(starter_type.env["EOS_CONFIG_DIR"]))
|
||||
assert name == f"eos-{config_dirs[-1].name}"
|
||||
starter = starter_type(None, xprocess)
|
||||
starter.wait(Mock())
|
||||
if startup_failure:
|
||||
raise RuntimeError("startup failed")
|
||||
return process.pid, "server.log"
|
||||
|
||||
xprocess.ensure.side_effect = ensure
|
||||
with pytest.raises(RuntimeError, match="failed"):
|
||||
with server_base(xprocess):
|
||||
raise RuntimeError("test body failed")
|
||||
|
||||
process.kill.assert_called_once_with()
|
||||
assert config_dirs and not config_dirs[0].exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reported_pid", [101, 999])
|
||||
def test_server_startup_check_rejects_another_server_on_the_same_port(
|
||||
cleanup_environment: SimpleNamespace,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
reported_pid: int,
|
||||
) -> None:
|
||||
"""A pre-existing server must never be mistaken for this test's process."""
|
||||
env = cleanup_environment
|
||||
process = env.make_process(101)
|
||||
xprocess = Mock()
|
||||
xprocess.getinfo.return_value.pid = process.pid
|
||||
monkeypatch.setattr("conftest.subprocess.run", Mock())
|
||||
monkeypatch.setattr("conftest.ProcessStarter.wait", Mock(return_value=True))
|
||||
response = Mock(status_code=200)
|
||||
response.json.return_value = {"pid": reported_pid}
|
||||
env.health.side_effect = None
|
||||
env.health.return_value = response
|
||||
|
||||
def ensure(name: str, starter_type: type[ProcessStarter]) -> tuple[int, str]:
|
||||
starter = starter_type(None, xprocess)
|
||||
starter.wait(Mock())
|
||||
assert starter.startup_check() is (reported_pid == process.pid)
|
||||
return process.pid, "server.log"
|
||||
|
||||
xprocess.ensure.side_effect = ensure
|
||||
with server_base(xprocess):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enumeration_denied", [False, True])
|
||||
def test_cleanup_terminates_real_owned_subprocess(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, enumeration_denied: bool
|
||||
) -> None:
|
||||
"""Exercise real process termination with available and denied enumeration."""
|
||||
connections = Mock(return_value=[])
|
||||
if enumeration_denied:
|
||||
connections.side_effect = psutil.AccessDenied(1)
|
||||
monkeypatch.setattr("conftest.psutil.net_connections", connections)
|
||||
monkeypatch.setattr("conftest.requests.get", Mock(side_effect=requests.ConnectionError))
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, "-c", "import sys; sys.stdin.buffer.read()"],
|
||||
stdin=subprocess.PIPE,
|
||||
)
|
||||
try:
|
||||
owned = psutil.Process(process.pid)
|
||||
cleanup_eos_eosdash(
|
||||
"127.0.0.1",
|
||||
8503,
|
||||
"127.0.0.1",
|
||||
8504,
|
||||
owned_processes=[owned],
|
||||
config_dir=str(tmp_path),
|
||||
)
|
||||
assert not owned.is_running()
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.kill()
|
||||
process.wait(timeout=5)
|
||||
if process.stdin is not None:
|
||||
process.stdin.close()
|
||||
Reference in New Issue
Block a user