Add a stats agent container: FastAPI + psutil behind an API key
Exposes host CPU, memory, disk, and network usage over HTTP for the dashboard's System Stats block (or anything else that can send a header) to poll. Auth is a required X-API-Key, auto-generated and persisted on first run if not supplied, checked with a constant-time comparison and rate-limited after repeated failures. No CORS, so a browser can't call it directly from another origin -- only a server-side caller can, keeping the key out of anyone's network tab.
This commit is contained in:
212
app/stats.py
Normal file
212
app/stats.py
Normal file
@@ -0,0 +1,212 @@
|
||||
"""Collects host stats.
|
||||
|
||||
Running inside a container normally only shows the container's own cgroup,
|
||||
not the host's — so getting real host numbers relies on the operator
|
||||
bind-mounting host paths in at conventional locations (see README.md):
|
||||
|
||||
-v /proc:/host/proc:ro
|
||||
-v /sys:/host/sys:ro
|
||||
-v /:/host/root:ro
|
||||
--pid host
|
||||
|
||||
Every mount is optional and auto-detected; whatever isn't mounted just
|
||||
falls back to the container's own view instead of failing outright.
|
||||
"""
|
||||
|
||||
import os
|
||||
import socket
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import psutil
|
||||
|
||||
HOST_PROC = "/host/proc" if os.path.isdir("/host/proc") else "/proc"
|
||||
HOST_ROOT = "/host/root" if os.path.isdir("/host/root") else None
|
||||
|
||||
# psutil reads /proc for cpu/memory/swap/boot-time/partition-list — pointing
|
||||
# it at the host's mounted procfs (rather than the container's own) is a
|
||||
# documented psutil feature, exactly for this containerized-monitoring case.
|
||||
psutil.PROCFS_PATH = HOST_PROC
|
||||
|
||||
# Pseudo/virtual filesystems that never represent real disk space — listing
|
||||
# these would just be noise (and some, like overlay, would double-count the
|
||||
# same underlying disk the container runtime already reports elsewhere).
|
||||
_SKIP_FSTYPES = {
|
||||
"proc", "sysfs", "cgroup", "cgroup2", "tmpfs", "devtmpfs", "devpts",
|
||||
"overlay", "squashfs", "fuse.lxcfs", "mqueue", "debugfs", "tracefs",
|
||||
"autofs", "nsfs", "rpc_pipefs", "binfmt_misc", "fusectl", "pstore",
|
||||
"bpf", "hugetlbfs", "securityfs", "efivarfs", "configfs", "ramfs",
|
||||
}
|
||||
|
||||
|
||||
def _hostname() -> str:
|
||||
override = os.environ.get("AGENT_HOSTNAME", "").strip()
|
||||
if override:
|
||||
return override
|
||||
if HOST_ROOT:
|
||||
try:
|
||||
name = (Path(HOST_ROOT) / "etc" / "hostname").read_text().strip()
|
||||
if name:
|
||||
return name
|
||||
except OSError:
|
||||
pass
|
||||
return socket.gethostname()
|
||||
|
||||
|
||||
def _cpu_model() -> str | None:
|
||||
try:
|
||||
with open(f"{HOST_PROC}/cpuinfo") as f:
|
||||
for line in f:
|
||||
if line.lower().startswith("model name"):
|
||||
return line.split(":", 1)[1].strip()
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _load_average() -> list[float]:
|
||||
# Read the host's loadavg file directly rather than via psutil.getloadavg
|
||||
# (which on some psutil versions shells out to the libc call instead of
|
||||
# honoring PROCFS_PATH, giving the container's own load instead).
|
||||
try:
|
||||
with open(f"{HOST_PROC}/loadavg") as f:
|
||||
parts = f.read().split()
|
||||
return [float(parts[0]), float(parts[1]), float(parts[2])]
|
||||
except (OSError, IndexError, ValueError):
|
||||
try:
|
||||
return list(psutil.getloadavg())
|
||||
except OSError:
|
||||
return [0.0, 0.0, 0.0]
|
||||
|
||||
|
||||
def _translate_mount(mountpoint: str) -> str:
|
||||
if not HOST_ROOT:
|
||||
return mountpoint
|
||||
if mountpoint == "/":
|
||||
return HOST_ROOT
|
||||
return os.path.join(HOST_ROOT, mountpoint.lstrip("/"))
|
||||
|
||||
|
||||
def _iter_mounts():
|
||||
# Deliberately not psutil.disk_partitions(): it reads
|
||||
# "{PROCFS_PATH}/self/mounts", and "self" is a magic symlink resolved
|
||||
# per-process by the kernel — which doesn't work through a bind mount of
|
||||
# someone else's /proc (there's no live procfs instance backing it, just
|
||||
# a copy of the directory tree), so it 404s as soon as PROCFS_PATH points
|
||||
# at a bind-mounted host /proc. "{PROCFS_PATH}/1/mounts" is a concrete,
|
||||
# non-magic path (host PID 1's mounts, i.e. the host's own mount table)
|
||||
# that works the same way whether /host/proc is a bind mount or not.
|
||||
for path in (f"{HOST_PROC}/1/mounts", f"{HOST_PROC}/mounts"):
|
||||
try:
|
||||
with open(path) as f:
|
||||
lines = f.readlines()
|
||||
except OSError:
|
||||
continue
|
||||
for line in lines:
|
||||
fields = line.split()
|
||||
if len(fields) < 3:
|
||||
continue
|
||||
device, mountpoint, fstype = fields[0], fields[1], fields[2]
|
||||
yield device, mountpoint, fstype
|
||||
return
|
||||
|
||||
|
||||
def _disks() -> list[dict]:
|
||||
disks = []
|
||||
seen_devices = set()
|
||||
for device, mountpoint, fstype in _iter_mounts():
|
||||
if not fstype or fstype in _SKIP_FSTYPES:
|
||||
continue
|
||||
if device in seen_devices:
|
||||
# The same device bind-mounted at multiple mountpoints (common
|
||||
# for docker's own overlay setup) would otherwise be counted
|
||||
# once per mountpoint.
|
||||
continue
|
||||
local_path = _translate_mount(mountpoint)
|
||||
try:
|
||||
usage = psutil.disk_usage(local_path)
|
||||
except OSError:
|
||||
continue
|
||||
seen_devices.add(device)
|
||||
disks.append(
|
||||
{
|
||||
"mountpoint": mountpoint,
|
||||
"device": device,
|
||||
"fstype": fstype,
|
||||
"totalBytes": usage.total,
|
||||
"usedBytes": usage.used,
|
||||
"freeBytes": usage.free,
|
||||
"percent": usage.percent,
|
||||
}
|
||||
)
|
||||
return sorted(disks, key=lambda d: d["mountpoint"])
|
||||
|
||||
|
||||
def _network() -> dict:
|
||||
# Plain /proc/net/dev is a symlink to /proc/self/net/dev — it always
|
||||
# reflects the *reading process's* network namespace, so it would only
|
||||
# ever show the container's own virtual interface even with host /proc
|
||||
# bind-mounted. /host/proc/1/net/dev is a concrete (non-symlink) path —
|
||||
# host PID 1's own counters, i.e. the host's real network namespace —
|
||||
# and, like the mounts file in _iter_mounts, is readable through the
|
||||
# bind mount without needing to share the host's PID namespace.
|
||||
pid1_path = f"{HOST_PROC}/1/net/dev"
|
||||
use_pid1 = os.path.exists(pid1_path)
|
||||
path = pid1_path if use_pid1 else f"{HOST_PROC}/net/dev"
|
||||
|
||||
interfaces = []
|
||||
try:
|
||||
with open(path) as f:
|
||||
lines = f.readlines()[2:]
|
||||
for line in lines:
|
||||
name, _, rest = line.partition(":")
|
||||
name = name.strip()
|
||||
if name == "lo":
|
||||
continue
|
||||
fields = rest.split()
|
||||
if len(fields) < 10:
|
||||
continue
|
||||
interfaces.append(
|
||||
{
|
||||
"name": name,
|
||||
"bytesRecv": int(fields[0]),
|
||||
"packetsRecv": int(fields[1]),
|
||||
"bytesSent": int(fields[8]),
|
||||
"packetsSent": int(fields[9]),
|
||||
}
|
||||
)
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
|
||||
return {"hostNamespace": use_pid1, "interfaces": interfaces}
|
||||
|
||||
|
||||
def collect() -> dict:
|
||||
cpu_percent = psutil.cpu_percent(interval=0.3)
|
||||
vm = psutil.virtual_memory()
|
||||
swap = psutil.swap_memory()
|
||||
boot_time = psutil.boot_time()
|
||||
|
||||
return {
|
||||
"hostname": _hostname(),
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"uptimeSeconds": int(time.time() - boot_time),
|
||||
"cpu": {
|
||||
"percent": cpu_percent,
|
||||
"cores": psutil.cpu_count(logical=True) or 0,
|
||||
"model": _cpu_model(),
|
||||
"loadAverage": _load_average(),
|
||||
},
|
||||
"memory": {
|
||||
"totalBytes": vm.total,
|
||||
"usedBytes": vm.used,
|
||||
"availableBytes": vm.available,
|
||||
"percent": vm.percent,
|
||||
"swapTotalBytes": swap.total,
|
||||
"swapUsedBytes": swap.used,
|
||||
"swapPercent": swap.percent,
|
||||
},
|
||||
"disks": _disks(),
|
||||
"network": _network(),
|
||||
}
|
||||
Reference in New Issue
Block a user