Mirrors/ProxMenux
1
0
Fork 0
mirror of https://github.com/MacRimi/ProxMenux.git synced 2026-11-01 18:32:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #561 from DrOwl/feature/zfs-pool-details

Browse Source 
Show ZFS pool layout, scrub status and device health in Storage view
This commit is contained in:
MacRimi
2026-10-30 21:38:12 +01:00
committed by GitHub
parent 89531bbab8 72491b2445
commit 668b557fd5
7 changed files with 1308 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AppImage/scripts/flask_server.py
+167 -18
View File
@@ -4031,6 +4031,149 @@ def _resolve_zfs_entry(entry):
return None
def get_zpool_status_details():
"""Return `zpool status` for every pool as parsed JSON, keyed by pool name.
Needs OpenZFS 2.3+ for `-j`. Returns None when the JSON output isn't
available (older ZFS, no zpool binary) so callers can tell "no details"
apart from "no pools". --json-int gives raw byte counts and epoch
timestamps; -P gives full device paths.
"""
try:
result = subprocess.run(
['zpool', 'status', '-j', '--json-int', '-P'],
capture_output=True, text=True, timeout=10
)
if result.returncode != 0:
return None
return json.loads(result.stdout).get('pools') or {}
except Exception:
return None
def _zfs_member_disk(path):
"""Map a pool member path (by-id, by-partlabel, /dev/sdX3...) to its base disk.
Pure sysfs (no lsblk subprocess) since /api/storage is polled and a
pool can have dozens of members. Non-/dev members (file vdevs) give None.
"""
if not path.startswith('/dev/'):
return None
try:
name = os.path.basename(os.path.realpath(path))
if os.path.exists(f'/sys/class/block/{name}/partition'):
return os.path.basename(os.path.dirname(os.path.realpath(f'/sys/class/block/{name}')))
if os.path.exists(f'/sys/class/block/{name}'):
return name
except Exception:
pass
return None
def _zfs_errors(node):
return {
'read_errors': node.get('read_errors', 0),
'write_errors': node.get('write_errors', 0),
'checksum_errors': node.get('checksum_errors', 0),
}
def _zfs_device(node, via=None):
path = node.get('path') or node.get('name', '')
return {
'path': path,
'disk': _zfs_member_disk(path),
'state': node.get('state', 'UNKNOWN'),
'slow_ios': node.get('slow_ios', 0),
'via': via,
**_zfs_errors(node),
}
_ZFS_PARITY_RE = re.compile(r'^(?:raidz|draid)(\d)')
def _zfs_vdev(node, vdev_class):
"""Flatten one top-level vdev into {type, parity, devices[...]}.
A leaf at the top level (single disk, stripe member, log/cache/spare
device) becomes a vdev of type 'disk' holding just itself. spare-N and
replacing-N wrappers inside a mirror/raidz are unwrapped, with each
device tagged `via` so the UI can show what's being swapped.
"""
children = node.get('vdevs') or {}
if not children:
device = _zfs_device(node)
return {
'name': device['path'],
'type': 'disk',
'parity': None,
'class': vdev_class,
'state': device['state'],
'devices': [device],
**_zfs_errors(node),
}
devices = []
for child in children.values():
if child.get('vdevs'):
devices.extend(_zfs_device(leaf, via=child.get('name')) for leaf in child['vdevs'].values())
else:
devices.append(_zfs_device(child))
parity = _ZFS_PARITY_RE.match(node.get('name', ''))
return {
'name': node.get('name', ''),
'type': node.get('vdev_type', ''),
'parity': int(parity.group(1)) if parity else None,
'class': vdev_class,
'state': node.get('state', 'UNKNOWN'),
'devices': devices,
**_zfs_errors(node),
}
def _zfs_pool_details(pool):
"""Turn one pool from `zpool status -j` into the shape the Storage view uses.
Data/special/dedup vdevs hang off the root vdev; log, cache and spare
devices live in separate top-level maps. GUIDs are dropped; they don't
fit in a JS number and the UI doesn't need them.
"""
root = (pool.get('vdevs') or {}).get(pool.get('name'), {})
vdevs = [_zfs_vdev(child, child.get('class', 'normal')) for child in (root.get('vdevs') or {}).values()]
for key, vdev_class in (('logs', 'log'), ('l2cache', 'cache'), ('spares', 'spare')):
vdevs.extend(_zfs_vdev(child, vdev_class) for child in (pool.get(key) or {}).values())
scan = None
stats = pool.get('scan_stats')
if stats:
# issued/to_examine drifts either side of 100 once finished (skipped
# blocks), so progress is only meaningful while the scan is running.
to_examine = stats.get('to_examine') or 0
running = stats.get('state') == 'SCANNING' and to_examine
scan = {
'function': stats.get('function'), # SCRUB | RESILVER
'state': stats.get('state'), # SCANNING | FINISHED | CANCELED
'start_time': stats.get('start_time'),
'end_time': stats.get('end_time'),
'errors': stats.get('errors', 0),
'repaired_bytes': stats.get('processed', 0),
'percent': min(100.0, round(100 * (stats.get('issued') or 0) / to_examine, 1)) if running else None,
}
def _tidy(text):
return ' '.join(text.split()) if text else None
return {
'status': _tidy(pool.get('status')),
'action': _tidy(pool.get('action')),
'error_count': pool.get('error_count', 0),
'scan': scan,
'vdevs': vdevs,
}
def get_system_disks():
"""
Detect which physical disks are used by the system (Proxmox).
@@ -4483,9 +4626,7 @@ def get_storage_info():
disk_partitions = psutil.disk_partitions()
total_used = 0
total_available = 0
zfs_disks = set()
for partition in disk_partitions:
try:
# Skip special filesystems
@@ -4531,9 +4672,11 @@ def get_storage_info():
continue
try:
result = subprocess.run(['zpool', 'list', '-H', '-p', '-o', 'name,size,alloc,free,health'],
result = subprocess.run(['zpool', 'list', '-H', '-p', '-o', 'name,size,alloc,free,health,frag,cap,dedup'],
capture_output=True, text=True, timeout=5)
if result.returncode == 0:
# One `zpool status` for all pools; None on ZFS < 2.3 (no -j)
zpool_details = get_zpool_status_details()
for line in result.stdout.strip().split('\n'):
if line:
parts = line.split('\t')
@@ -4543,6 +4686,14 @@ def get_storage_info():
pool_alloc_bytes = int(parts[2])
pool_free_bytes = int(parts[3])
pool_health = parts[4]
# frag is '-' on pools without spacemap histograms
extra = parts[5:8] + ['-'] * (8 - len(parts))
fragmentation = int(extra[0]) if extra[0].isdigit() else None
capacity = int(extra[1]) if extra[1].isdigit() else None
try:
dedup_ratio = float(extra[2])
except ValueError:
dedup_ratio = None
total_used += pool_alloc_bytes
total_available += pool_free_bytes
@@ -4563,22 +4714,20 @@ def get_storage_info():
'size': format_zfs_size(pool_size_bytes),
'allocated': format_zfs_size(pool_alloc_bytes),
'free': format_zfs_size(pool_free_bytes),
'health': pool_health
'health': pool_health,
'fragmentation': fragmentation,
'capacity': capacity,
'dedup_ratio': dedup_ratio,
'details_available': False,
}
if zpool_details and pool_name in zpool_details:
try:
pool_info.update(_zfs_pool_details(zpool_details[pool_name]))
pool_info['details_available'] = True
except Exception:
pass
storage_data['zfs_pools'].append(pool_info)
try:
pool_status = subprocess.run(['zpool', 'status', pool_name],
capture_output=True, text=True, timeout=5)
if pool_status.returncode == 0:
for status_line in pool_status.stdout.split('\n'):
for disk_name in physical_disks.keys():
if disk_name in status_line:
zfs_disks.add(disk_name)
except Exception as e:
# print(f"Error getting ZFS pool status for {pool_name}: {e}")
pass
except FileNotFoundError:
# print("[v0] Note: ZFS not installed")
pass
AppImage/scripts/tests/fixtures/zpool_status_degraded_raidz1.json
Vendored
+140
View File
@@ -0,0 +1,140 @@
{
"output_version": {
"command": "zpool status",
"vers_major": 0,
"vers_minor": 1
},
"pools": {
"pmx-test": {
"name": "pmx-test",
"state": "DEGRADED",
"txg": 8,
"spa_version": 5000,
"zpl_version": 5,
"status": "One or more devices has been taken offline by the administrator.\n\tSufficient replicas exist for the pool to continue functioning in a\n\tdegraded state.\n",
"action": "Online the device using 'zpool online' or replace the device with\n\t'zpool replace'.\n",
"vdevs": {
"pmx-test": {
"name": "pmx-test",
"vdev_type": "root",
"class": "normal",
"state": "DEGRADED",
"alloc_space": 265216,
"total_space": 738197504,
"def_space": 491651072,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"vdevs": {
"raidz1-0": {
"name": "raidz1-0",
"vdev_type": "raidz",
"class": "normal",
"state": "DEGRADED",
"alloc_space": 265216,
"total_space": 738197504,
"def_space": 491651072,
"rep_dev_size": 738197504,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"vdevs": {
"/root/zfstest/d1.img": {
"name": "/root/zfstest/d1.img",
"vdev_type": "file",
"path": "/root/zfstest/d1.img",
"class": "normal",
"state": "ONLINE",
"rep_dev_size": 250784427,
"phys_space": 268435456,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"slow_ios": 0,
"guid": 1001
},
"/root/zfstest/d2.img": {
"name": "/root/zfstest/d2.img",
"vdev_type": "file",
"path": "/root/zfstest/d2.img",
"class": "normal",
"state": "OFFLINE",
"rep_dev_size": 250784427,
"phys_space": 268435456,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"slow_ios": 0,
"guid": 1001
},
"/root/zfstest/d3.img": {
"name": "/root/zfstest/d3.img",
"vdev_type": "file",
"path": "/root/zfstest/d3.img",
"class": "normal",
"state": "ONLINE",
"rep_dev_size": 250784427,
"phys_space": 268435456,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"slow_ios": 0,
"guid": 1001
}
},
"guid": 1001
}
},
"guid": 1001
}
},
"logs": {
"/root/zfstest/d4.img": {
"name": "/root/zfstest/d4.img",
"vdev_type": "file",
"path": "/root/zfstest/d4.img",
"class": "log",
"state": "ONLINE",
"total_space": 251658240,
"def_space": 251658240,
"rep_dev_size": 256376832,
"phys_space": 268435456,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"slow_ios": 0,
"guid": 1001
}
},
"l2cache": {
"/root/zfstest/d5.img": {
"name": "/root/zfstest/d5.img",
"vdev_type": "file",
"path": "/root/zfstest/d5.img",
"class": "l2cache",
"state": "ONLINE",
"total_space": 263716352,
"rep_dev_size": 268435456,
"phys_space": 268435456,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"slow_ios": 0,
"guid": 1001
}
},
"spares": {
"/root/zfstest/d6.img": {
"name": "/root/zfstest/d6.img",
"vdev_type": "file",
"path": "/root/zfstest/d6.img",
"class": "spare",
"state": "AVAIL",
"guid": 1001
}
},
"error_count": 0,
"pool_guid": 1000
}
}
}
AppImage/scripts/tests/fixtures/zpool_status_multi_pool.json
Vendored
+399
View File
@@ -0,0 +1,399 @@
{
"output_version": {
"command": "zpool status",
"vers_major": 0,
"vers_minor": 1
},
"pools": {
"rpool": {
"name": "rpool",
"state": "ONLINE",
"txg": 25710203,
"spa_version": 5000,
"zpl_version": 5,
"scan_stats": {
"function": "SCRUB",
"state": "FINISHED",
"start_time": 1791674641,
"end_time": 1791675294,
"to_examine": 421980536832,
"examined": 421980536832,
"skipped": 22963814,
"processed": 0,
"errors": 0,
"bytes_per_scan": 0,
"pass_start": 1791674660,
"scrub_pause": 0,
"scrub_spent_paused": 0,
"issued_bytes_per_scan": 421980536832,
"issued": 421980536832
},
"vdevs": {
"rpool": {
"name": "rpool",
"vdev_type": "root",
"class": "normal",
"state": "ONLINE",
"alloc_space": 411243118592,
"total_space": 996432412672,
"def_space": 996432412672,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"vdevs": {
"mirror-0": {
"name": "mirror-0",
"vdev_type": "mirror",
"class": "normal",
"state": "ONLINE",
"alloc_space": 411243118592,
"total_space": 996432412672,
"def_space": 996432412672,
"rep_dev_size": 996432412672,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"vdevs": {
"nvme-CT1000P5PSSD8_2231000000A1-part3": {
"name": "nvme-CT1000P5PSSD8_2231000000A1-part3",
"vdev_type": "disk",
"path": "/dev/disk/by-id/nvme-CT1000P5PSSD8_2231000000A1-part3",
"class": "normal",
"state": "ONLINE",
"rep_dev_size": 996432412672,
"phys_space": 999653638144,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"slow_ios": 0,
"guid": 1001
},
"nvme-s0-1tb-open": {
"name": "nvme-s0-1tb-open",
"vdev_type": "disk",
"path": "/dev/disk/by-partlabel/nvme-s0-1tb-open",
"class": "normal",
"state": "ONLINE",
"rep_dev_size": 996432412672,
"phys_space": 999653638144,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"slow_ios": 0,
"guid": 1001
}
},
"guid": 1001
}
},
"guid": 1001
}
},
"error_count": 0,
"pool_guid": 1000
},
"tank-m1": {
"name": "tank-m1",
"state": "ONLINE",
"txg": 32213752,
"spa_version": 5000,
"zpl_version": 5,
"scan_stats": {
"function": "SCRUB",
"state": "FINISHED",
"start_time": 1791674641,
"end_time": 1791675294,
"to_examine": 528280977408,
"examined": 529354719232,
"skipped": 56623104,
"processed": 0,
"errors": 0,
"bytes_per_scan": 0,
"pass_start": 1791674757,
"scrub_pause": 0,
"scrub_spent_paused": 0,
"issued_bytes_per_scan": 507879882752,
"issued": 528280977408
},
"vdevs": {
"tank-m1": {
"name": "tank-m1",
"vdev_type": "root",
"class": "normal",
"state": "ONLINE",
"alloc_space": 525059751936,
"total_space": 996432412672,
"def_space": 996432412672,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"vdevs": {
"mirror-0": {
"name": "mirror-0",
"vdev_type": "mirror",
"class": "normal",
"state": "ONLINE",
"alloc_space": 525059751936,
"total_space": 996432412672,
"def_space": 996432412672,
"rep_dev_size": 996432412672,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"vdevs": {
"ssd-t1-1Tb-mx500": {
"name": "ssd-t1-1Tb-mx500",
"vdev_type": "disk",
"path": "/dev/disk/by-partlabel/ssd-t1-1Tb-mx500",
"class": "normal",
"state": "ONLINE",
"rep_dev_size": 996432412672,
"phys_space": 1000727379968,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"slow_ios": 0,
"guid": 1001
},
"ssd-t2-1Tb-mx500": {
"name": "ssd-t2-1Tb-mx500",
"vdev_type": "disk",
"path": "/dev/disk/by-partlabel/ssd-t2-1Tb-mx500",
"class": "normal",
"state": "ONLINE",
"rep_dev_size": 996432412672,
"phys_space": 1000727379968,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"slow_ios": 69,
"guid": 1001
}
},
"guid": 1001
}
},
"guid": 1001
}
},
"error_count": 0,
"pool_guid": 1000
},
"tank-rz2": {
"name": "tank-rz2",
"state": "ONLINE",
"txg": 7867809,
"spa_version": 5000,
"zpl_version": 5,
"scan_stats": {
"function": "SCRUB",
"state": "FINISHED",
"start_time": 1791674641,
"end_time": 1791675294,
"to_examine": 10511331161538,
"examined": 10522326277816,
"skipped": 10297184092,
"processed": 0,
"errors": 0,
"bytes_per_scan": 0,
"pass_start": 1791674783,
"scrub_pause": 0,
"scrub_spent_paused": 0,
"issued_bytes_per_scan": 10500336045260,
"issued": 10500336045260
},
"vdevs": {
"tank-rz2": {
"name": "tank-rz2",
"vdev_type": "root",
"class": "normal",
"state": "ONLINE",
"alloc_space": 12094627905536,
"total_space": 19241453486080,
"def_space": 11324969766092,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"vdevs": {
"raidz2-0": {
"name": "raidz2-0",
"vdev_type": "raidz",
"class": "normal",
"state": "ONLINE",
"alloc_space": 12094627905536,
"total_space": 19241453486080,
"def_space": 11324969766092,
"rep_dev_size": 19241453486080,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"vdevs": {
"ssd-3T5-Tosh-B001": {
"name": "ssd-3T5-Tosh-B001",
"vdev_type": "disk",
"path": "/dev/disk/by-partlabel/ssd-3T5-Tosh-B001",
"class": "normal",
"state": "ONLINE",
"rep_dev_size": 3837295580938,
"phys_space": 3837295580938,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"slow_ios": 0,
"guid": 1001
},
"ssd-3T5-Tosh-B002": {
"name": "ssd-3T5-Tosh-B002",
"vdev_type": "disk",
"path": "/dev/disk/by-partlabel/ssd-3T5-Tosh-B002",
"class": "normal",
"state": "ONLINE",
"rep_dev_size": 3837295580938,
"phys_space": 3837295580938,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"slow_ios": 0,
"guid": 1001
},
"ssd-3T5-Tosh-B003": {
"name": "ssd-3T5-Tosh-B003",
"vdev_type": "disk",
"path": "/dev/disk/by-partlabel/ssd-3T5-Tosh-B003",
"class": "normal",
"state": "ONLINE",
"rep_dev_size": 3837295580938,
"phys_space": 3837295580938,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"slow_ios": 0,
"guid": 1001
},
"ssd-3T5-Tosh-B004": {
"name": "ssd-3T5-Tosh-B004",
"vdev_type": "disk",
"path": "/dev/disk/by-partlabel/ssd-3T5-Tosh-B004",
"class": "normal",
"state": "ONLINE",
"rep_dev_size": 3837295580938,
"phys_space": 3837295580938,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"slow_ios": 0,
"guid": 1001
},
"ssd-3T5-Tosh-B005": {
"name": "ssd-3T5-Tosh-B005",
"vdev_type": "disk",
"path": "/dev/disk/by-partlabel/ssd-3T5-Tosh-B005",
"class": "normal",
"state": "ONLINE",
"rep_dev_size": 3837295580938,
"phys_space": 3837295580938,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"slow_ios": 0,
"guid": 1001
}
},
"guid": 1001
}
},
"guid": 1001
}
},
"error_count": 0,
"pool_guid": 1000
},
"tank-spin-1": {
"name": "tank-spin-1",
"state": "ONLINE",
"txg": 23263182,
"spa_version": 5000,
"zpl_version": 5,
"scan_stats": {
"function": "SCRUB",
"state": "FINISHED",
"start_time": 1791674641,
"end_time": 1791675294,
"to_examine": 5761440929546,
"examined": 5761440929546,
"skipped": 358629769216,
"processed": 0,
"errors": 0,
"bytes_per_scan": 0,
"pass_start": 1791674909,
"scrub_pause": 0,
"scrub_spent_paused": 0,
"issued_bytes_per_scan": 5398602092380,
"issued": 5398602092380
},
"vdevs": {
"tank-spin-1": {
"name": "tank-spin-1",
"vdev_type": "root",
"class": "normal",
"state": "ONLINE",
"alloc_space": 5662484883046,
"total_space": 7850513022320,
"def_space": 7850513022320,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"vdevs": {
"mirror-0": {
"name": "mirror-0",
"vdev_type": "mirror",
"class": "normal",
"state": "ONLINE",
"alloc_space": 5662484883046,
"total_space": 7850513022320,
"def_space": 7850513022320,
"rep_dev_size": 7850513022320,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"vdevs": {
"wwn-0x5000c50000000001-part1": {
"name": "wwn-0x5000c50000000001-part1",
"vdev_type": "disk",
"path": "/dev/disk/by-id/wwn-0x5000c50000000001-part1",
"class": "normal",
"state": "ONLINE",
"rep_dev_size": 7850513022320,
"phys_space": 7861508138598,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"slow_ios": 0,
"guid": 1001
},
"wwn-0x5000c50000000002-part1": {
"name": "wwn-0x5000c50000000002-part1",
"vdev_type": "disk",
"path": "/dev/disk/by-id/wwn-0x5000c50000000002-part1",
"class": "normal",
"state": "ONLINE",
"rep_dev_size": 7850513022320,
"phys_space": 7861508138598,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"slow_ios": 0,
"guid": 1001
}
},
"guid": 1001
}
},
"guid": 1001
}
},
"error_count": 0,
"pool_guid": 1000
}
}
}
AppImage/scripts/tests/fixtures/zpool_status_single_rpool.json
Vendored
+69
View File
@@ -0,0 +1,69 @@
{
"output_version": {
"command": "zpool status",
"vers_major": 0,
"vers_minor": 1
},
"pools": {
"rpool": {
"name": "rpool",
"state": "ONLINE",
"txg": 38071404,
"spa_version": 5000,
"zpl_version": 5,
"scan_stats": {
"function": "SCRUB",
"state": "FINISHED",
"start_time": 1791674641,
"end_time": 1791674884,
"to_examine": 36167286784,
"examined": 36193234944,
"skipped": 6365184,
"processed": 0,
"errors": 0,
"bytes_per_scan": 0,
"pass_start": 1791909484,
"scrub_pause": 0,
"scrub_spent_paused": 0,
"issued_bytes_per_scan": 0,
"issued": 36186869760
},
"vdevs": {
"rpool": {
"name": "rpool",
"vdev_type": "root",
"class": "normal",
"state": "ONLINE",
"alloc_space": 37495803904,
"total_space": 476741369856,
"def_space": 476741369856,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"vdevs": {
"/dev/disk/by-id/ata-Patriot_Burst_00000000000000000001-part3": {
"name": "/dev/disk/by-id/ata-Patriot_Burst_00000000000000000001-part3",
"vdev_type": "disk",
"path": "/dev/disk/by-id/ata-Patriot_Burst_00000000000000000001-part3",
"class": "normal",
"state": "ONLINE",
"alloc_space": 37495803904,
"total_space": 476741369856,
"def_space": 476741369856,
"rep_dev_size": 476746088448,
"phys_space": 479565971456,
"read_errors": 0,
"write_errors": 0,
"checksum_errors": 0,
"slow_ios": 0,
"guid": 1001
}
},
"guid": 1001
}
},
"error_count": 0,
"pool_guid": 1000
}
}
}
AppImage/scripts/tests/test_zfs_pool_details.py
+166
View File
@@ -0,0 +1,166 @@
"""ZFS pool layout parsing for the Storage view; never imports flask_server.
Fixtures are real `zpool status -j --json-int -P` output from PVE 9 /
OpenZFS 2.4.4 hosts, with disk serials and WWNs replaced:
- zpool_status_multi_pool.json: mirror, mirror, raidz2, mirror (one member with
slow I/Os). Captured without --json-int and normalised to integers.
- zpool_status_single_rpool.json: single disk directly under the root vdev.
- zpool_status_degraded_raidz1.json: file-backed raidz1 with one member
offlined, plus log, cache and spare devices; never scrubbed.
Run: python3 -m unittest discover -s AppImage/scripts/tests -p test_zfs_pool_details.py -v
"""
import ast
import copy
import json
import posixpath
import re
from pathlib import Path
from types import SimpleNamespace
import unittest
SOURCE = Path(__file__).resolve().parents[1] / "flask_server.py"
FIXTURES = Path(__file__).resolve().parent / "fixtures"
NAMES = {"_zfs_member_disk", "_zfs_errors", "_zfs_device", "_ZFS_PARITY_RE",
"_zfs_vdev", "_zfs_pool_details"}
def fake_os(links=None, existing=()):
"""Minimal `os` stand-in: realpath via a symlink map, exists via a set."""
links = links or {}
return SimpleNamespace(path=SimpleNamespace(
realpath=lambda p: links.get(p, p),
exists=lambda p: p in existing,
basename=posixpath.basename,
dirname=posixpath.dirname,
))
def load(os_module=None):
tree = ast.parse(SOURCE.read_text())
nodes = [n for n in tree.body
if (isinstance(n, ast.FunctionDef) and n.name in NAMES)
or (isinstance(n, ast.Assign) and any(getattr(t, "id", None) in NAMES for t in n.targets))]
found = {n.name if isinstance(n, ast.FunctionDef) else n.targets[0].id for n in nodes}
assert found == NAMES, NAMES - found
env = {"os": os_module or fake_os(), "re": re}
exec(compile(ast.Module(body=nodes, type_ignores=[]), str(SOURCE), "exec"), env)
return env
def pools(name):
return json.loads((FIXTURES / f"{name}.json").read_text())["pools"]
class ZfsPoolDetailsTests(unittest.TestCase):
def setUp(self):
self.details = load()["_zfs_pool_details"]
def test_degraded_raidz1_with_log_cache_and_spare(self):
pool = self.details(pools("zpool_status_degraded_raidz1")["pmx-test"])
self.assertEqual([v["class"] for v in pool["vdevs"]], ["normal", "log", "cache", "spare"])
raidz = pool["vdevs"][0]
self.assertEqual((raidz["type"], raidz["parity"], raidz["state"]), ("raidz", 1, "DEGRADED"))
self.assertEqual([d["state"] for d in raidz["devices"]], ["ONLINE", "OFFLINE", "ONLINE"])
self.assertEqual(pool["vdevs"][3]["state"], "AVAIL")
self.assertIsNone(pool["scan"])
self.assertTrue(pool["status"].startswith("One or more devices has been taken offline"))
self.assertNotIn("\t", pool["status"])
self.assertNotIn("\n", pool["action"])
# File vdevs have no block device behind them
self.assertIsNone(raidz["devices"][0]["disk"])
def test_single_disk_pool_is_a_disk_vdev(self):
pool = self.details(pools("zpool_status_single_rpool")["rpool"])
self.assertEqual(len(pool["vdevs"]), 1)
vdev = pool["vdevs"][0]
self.assertEqual((vdev["type"], vdev["class"], vdev["parity"]), ("disk", "normal", None))
self.assertEqual(len(vdev["devices"]), 1)
self.assertEqual(vdev["name"], vdev["devices"][0]["path"])
def test_finished_scrub_has_no_progress_percentage(self):
pool = self.details(pools("zpool_status_single_rpool")["rpool"])
self.assertEqual((pool["scan"]["function"], pool["scan"]["state"]), ("SCRUB", "FINISHED"))
self.assertEqual(pool["scan"]["end_time"], 1791674884)
# issued/to_examine is 100.1% for this real scrub; only meaningful while running
self.assertIsNone(pool["scan"]["percent"])
def test_mirror_raidz2_and_slow_ios(self):
multi = pools("zpool_status_multi_pool")
parsed = {name: self.details(pool) for name, pool in multi.items()}
self.assertEqual(parsed["tank-rz2"]["vdevs"][0]["parity"], 2)
self.assertEqual(len(parsed["tank-rz2"]["vdevs"][0]["devices"]), 5)
for name in ("rpool", "tank-m1", "tank-spin-1"):
self.assertEqual(parsed[name]["vdevs"][0]["type"], "mirror", name)
slow = [d["slow_ios"] for d in parsed["tank-m1"]["vdevs"][0]["devices"]]
self.assertEqual(slow, [0, 69])
# Member paths come from `path`, not the JSON key (which may drop -partN)
paths = [d["path"] for p in parsed.values() for v in p["vdevs"] for d in v["devices"]]
self.assertTrue(all(p.startswith("/dev/disk/") for p in paths), paths)
def test_resilver_with_spare_swapping_in(self):
# Synthetic: the degraded fixture with d2 FAULTED and the spare
# attached under spare-1, as `zpool status` shows during a resilver.
raw = copy.deepcopy(pools("zpool_status_degraded_raidz1")["pmx-test"])
members = raw["vdevs"]["pmx-test"]["vdevs"]["raidz1-0"]["vdevs"]
failed = members.pop("/root/zfstest/d2.img")
failed.update(state="FAULTED", read_errors=12)
spare = dict(raw["spares"]["/root/zfstest/d6.img"], state="ONLINE")
members["spare-1"] = {"name": "spare-1", "vdev_type": "spare", "state": "DEGRADED",
"vdevs": {"d2": failed, "d6": spare}}
raw["spares"]["/root/zfstest/d6.img"]["state"] = "INUSE"
raw["scan_stats"] = {"function": "RESILVER", "state": "SCANNING", "start_time": 1,
"end_time": 0, "to_examine": 1000, "issued": 425, "errors": 0,
"processed": 0}
pool = self.details(raw)
devices = pool["vdevs"][0]["devices"]
self.assertEqual([d["via"] for d in devices], [None, None, "spare-1", "spare-1"])
self.assertEqual(devices[2]["read_errors"], 12)
self.assertEqual(pool["scan"]["percent"], 42.5)
self.assertEqual(pool["vdevs"][-1]["state"], "INUSE")
def test_output_is_json_serialisable_without_guids(self):
for name, pool in pools("zpool_status_multi_pool").items():
text = json.dumps(self.details(pool))
self.assertNotIn("guid", text, name)
class ZfsMemberDiskTests(unittest.TestCase):
def test_partition_resolves_to_parent_disk(self):
member_disk = load(fake_os(
links={"/dev/disk/by-partlabel/ssd-t1": "/dev/sdf1",
"/sys/class/block/sdf1": "/sys/devices/pci0000:00/0000:02:00.1/ata1/host0/target0:0:0/0:0:0:0/block/sdf/sdf1"},
existing={"/sys/class/block/sdf1/partition"},
))["_zfs_member_disk"]
self.assertEqual(member_disk("/dev/disk/by-partlabel/ssd-t1"), "sdf")
def test_nvme_partition_resolves_to_namespace(self):
member_disk = load(fake_os(
links={"/dev/disk/by-id/nvme-X-part3": "/dev/nvme0n1p3",
"/sys/class/block/nvme0n1p3": "/sys/devices/pci0000:00/0000:08:00.0/nvme/nvme0/nvme0n1/nvme0n1p3"},
existing={"/sys/class/block/nvme0n1p3/partition"},
))["_zfs_member_disk"]
self.assertEqual(member_disk("/dev/disk/by-id/nvme-X-part3"), "nvme0n1")
def test_whole_disk_member(self):
member_disk = load(fake_os(
links={"/dev/disk/by-id/wwn-0x1": "/dev/sdj"},
existing={"/sys/class/block/sdj"},
))["_zfs_member_disk"]
self.assertEqual(member_disk("/dev/disk/by-id/wwn-0x1"), "sdj")
def test_missing_device_and_file_vdev_give_none(self):
member_disk = load(fake_os())["_zfs_member_disk"]
self.assertIsNone(member_disk("/dev/disk/by-id/gone"))
self.assertIsNone(member_disk("/root/zfstest/d1.img"))
if __name__ == "__main__":
unittest.main()