Add resource limits, abuse protection and overload handling
Running games always continue; new work is refused when there is no
capacity; abuse is rejected immediately.
- Docker: CPU/memory/pids caps per container (overridable in .env);
uvicorn message size, queue and concurrency limits
- Server abuse limits: connections total/per IP (close 1013), message
rate per connection (token bucket, close 1008 on floods), lobby
creation rate per IP
- Load monitor: event loop lag, CPU of the own cgroup quota and
throttling, with hysteresis (normal/busy/overloaded, severe lag
overloads immediately)
- Admission control: create_lobby and start_game refused while
overloaded or MAX_ACTIVE_GAMES are running; server_status message
broadcast on changes; /health reports load but stays ok
- AI bridge: timeout sent with requests, in-flight cap, circuit breaker
- AI service: bounded workers and queue ("overloaded"), drops expired
requests, degrades to the heuristic agent when busy or too slow
- Website: busy notices, Create/Start disabled when not accepting
games, slow retry when the server is full
- Tests for limiters, load levels, admission control and AI service
limits; docs updated
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,248 @@
|
||||
"""Load monitoring: detects when the server runs out of compute.
|
||||
|
||||
Every ``load_sample_interval`` seconds the monitor measures
|
||||
|
||||
- **event loop lag**: how much later than scheduled the loop woke up. This is
|
||||
the most direct signal that the server is slow for players;
|
||||
- **CPU usage** of the process relative to what it may use (the container's
|
||||
CPU quota, at most one core because Python runs game logic on one core);
|
||||
- **CPU throttling** from cgroup v2 (``/sys/fs/cgroup/cpu.stat``): the share
|
||||
of scheduling periods in which the container hit its CPU limit.
|
||||
|
||||
Each sample is classified as normal / busy / overloaded, and ``next_level``
|
||||
turns the recent classifications into the current level with hysteresis, so a
|
||||
single slow second does not flip the state back and forth.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from collections import deque
|
||||
from collections.abc import Callable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
|
||||
from .config import Settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CGROUP_DIR = Path("/sys/fs/cgroup")
|
||||
|
||||
|
||||
class LoadLevel(str, Enum):
|
||||
NORMAL = "normal"
|
||||
BUSY = "busy"
|
||||
OVERLOADED = "overloaded"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LoadSample:
|
||||
loop_lag: float
|
||||
# Share of the available CPU used (None if unknown).
|
||||
cpu: float | None = None
|
||||
# Share of CPU periods in which the container was throttled (None if unknown).
|
||||
throttled: float | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LoadThresholds:
|
||||
lag_busy: float
|
||||
lag_overload: float
|
||||
lag_severe: float
|
||||
cpu_busy: float
|
||||
cpu_overload: float
|
||||
throttle_overload: float
|
||||
overload_samples: int
|
||||
recover_samples: int
|
||||
|
||||
@classmethod
|
||||
def from_settings(cls, settings: Settings) -> LoadThresholds:
|
||||
return cls(
|
||||
lag_busy=settings.load_lag_busy,
|
||||
lag_overload=settings.load_lag_overload,
|
||||
lag_severe=settings.load_lag_severe,
|
||||
cpu_busy=settings.load_cpu_busy,
|
||||
cpu_overload=settings.load_cpu_overload,
|
||||
throttle_overload=settings.load_throttle_overload,
|
||||
overload_samples=max(1, settings.load_overload_samples),
|
||||
recover_samples=max(1, settings.load_recover_samples),
|
||||
)
|
||||
|
||||
|
||||
def classify(sample: LoadSample, t: LoadThresholds) -> LoadLevel:
|
||||
"""Level of a single sample."""
|
||||
if (
|
||||
sample.loop_lag >= t.lag_overload
|
||||
or (sample.cpu is not None and sample.cpu >= t.cpu_overload)
|
||||
or (sample.throttled is not None and sample.throttled >= t.throttle_overload)
|
||||
):
|
||||
return LoadLevel.OVERLOADED
|
||||
if sample.loop_lag >= t.lag_busy or (sample.cpu is not None and sample.cpu >= t.cpu_busy):
|
||||
return LoadLevel.BUSY
|
||||
return LoadLevel.NORMAL
|
||||
|
||||
|
||||
def next_level(
|
||||
current: LoadLevel, recent: Sequence[LoadLevel], t: LoadThresholds, severe: bool = False
|
||||
) -> LoadLevel:
|
||||
"""Current level from the recent sample levels (oldest first), with hysteresis.
|
||||
|
||||
- Overloaded after ``overload_samples`` overloaded samples in a row, or at
|
||||
once when the latest sample was ``severe`` (extreme event loop lag).
|
||||
- Leaving overload (and busy) requires ``recover_samples`` samples in a row
|
||||
below that level.
|
||||
- Busy is entered immediately (it is only a hint for players).
|
||||
"""
|
||||
if severe:
|
||||
return LoadLevel.OVERLOADED
|
||||
if not recent:
|
||||
return current
|
||||
last_bad = recent[-t.overload_samples:]
|
||||
if len(last_bad) == t.overload_samples and all(level is LoadLevel.OVERLOADED for level in last_bad):
|
||||
return LoadLevel.OVERLOADED
|
||||
|
||||
window = recent[-t.recover_samples:]
|
||||
full_window = len(window) == t.recover_samples
|
||||
if current is LoadLevel.OVERLOADED and not (
|
||||
full_window and all(level is not LoadLevel.OVERLOADED for level in window)
|
||||
):
|
||||
return LoadLevel.OVERLOADED
|
||||
if recent[-1] is not LoadLevel.NORMAL:
|
||||
return LoadLevel.BUSY
|
||||
if current is not LoadLevel.NORMAL and not (full_window and all(level is LoadLevel.NORMAL for level in window)):
|
||||
return LoadLevel.BUSY
|
||||
return LoadLevel.NORMAL
|
||||
|
||||
|
||||
# --- cgroup v2 helpers ------------------------------------------------------
|
||||
|
||||
|
||||
def parse_cpu_max(text: str) -> float | None:
|
||||
"""CPU quota in cores from ``cpu.max`` ("max 100000" = unlimited)."""
|
||||
parts = text.split()
|
||||
if len(parts) != 2 or parts[0] == "max":
|
||||
return None
|
||||
try:
|
||||
return int(parts[0]) / int(parts[1])
|
||||
except (ValueError, ZeroDivisionError):
|
||||
return None
|
||||
|
||||
|
||||
def parse_cpu_stat(text: str) -> dict[str, int]:
|
||||
stats: dict[str, int] = {}
|
||||
for line in text.splitlines():
|
||||
key, _, value = line.partition(" ")
|
||||
if value.strip().isdigit():
|
||||
stats[key] = int(value)
|
||||
return stats
|
||||
|
||||
|
||||
def own_cgroup_dir(root: Path = CGROUP_DIR, proc_cgroup: Path = Path("/proc/self/cgroup")) -> Path:
|
||||
"""The cgroup v2 directory of this process.
|
||||
|
||||
In a container this is ``/sys/fs/cgroup`` itself (cgroup namespace); on a
|
||||
host it is a sub directory such as ``/sys/fs/cgroup/user.slice/…scope``.
|
||||
"""
|
||||
for line in (_read(proc_cgroup) or "").splitlines():
|
||||
if line.startswith("0::"):
|
||||
candidate = root / line[3:].strip().lstrip("/")
|
||||
if candidate.is_dir():
|
||||
return candidate
|
||||
return root
|
||||
|
||||
|
||||
def _read(path: Path) -> str | None:
|
||||
try:
|
||||
return path.read_text()
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
class LoadMonitor:
|
||||
def __init__(
|
||||
self,
|
||||
settings: Settings,
|
||||
on_change: Callable[[LoadLevel], None] | None = None,
|
||||
cgroup_dir: Path | None = None,
|
||||
) -> None:
|
||||
self._settings = settings
|
||||
self._thresholds = LoadThresholds.from_settings(settings)
|
||||
self._on_change = on_change
|
||||
self._cgroup_dir = cgroup_dir if cgroup_dir is not None else own_cgroup_dir()
|
||||
self._forced = LoadLevel(settings.load_force_level) if settings.load_force_level else None
|
||||
self._level = LoadLevel.NORMAL
|
||||
self._recent: deque[LoadLevel] = deque(
|
||||
maxlen=max(self._thresholds.overload_samples, self._thresholds.recover_samples)
|
||||
)
|
||||
self.last_sample = LoadSample(loop_lag=0.0)
|
||||
|
||||
quota = parse_cpu_max(_read(self._cgroup_dir / "cpu.max") or "")
|
||||
# Game logic runs on one core, so more than one core cannot be used.
|
||||
self._cpu_capacity = min(quota, 1.0) if quota else 1.0
|
||||
self._last_cpu_time = time.process_time()
|
||||
self._last_wall = time.monotonic()
|
||||
self._last_cgroup = self._cgroup_stats()
|
||||
|
||||
@property
|
||||
def level(self) -> LoadLevel:
|
||||
return self._forced or self._level
|
||||
|
||||
def record(self, sample: LoadSample) -> None:
|
||||
"""Add a sample and update the level (called by ``run``; public for tests)."""
|
||||
self.last_sample = sample
|
||||
self._recent.append(classify(sample, self._thresholds))
|
||||
severe = sample.loop_lag >= self._thresholds.lag_severe
|
||||
new_level = next_level(self._level, list(self._recent), self._thresholds, severe)
|
||||
if new_level is self._level:
|
||||
return
|
||||
logger.log(
|
||||
logging.WARNING if new_level is LoadLevel.OVERLOADED else logging.INFO,
|
||||
"Load level %s -> %s (lag %.3fs, cpu %s, throttled %s)",
|
||||
self._level.value, new_level.value, sample.loop_lag, sample.cpu, sample.throttled,
|
||||
)
|
||||
self._level = new_level
|
||||
if self._on_change is not None and self._forced is None:
|
||||
self._on_change(new_level)
|
||||
|
||||
async def run(self) -> None:
|
||||
loop = asyncio.get_running_loop()
|
||||
interval = self._settings.load_sample_interval
|
||||
while True:
|
||||
started = loop.time()
|
||||
await asyncio.sleep(interval)
|
||||
lag = max(0.0, loop.time() - started - interval)
|
||||
try:
|
||||
self.record(self._measure(lag))
|
||||
except Exception:
|
||||
logger.exception("Load measurement failed")
|
||||
|
||||
def stats(self) -> dict[str, object]:
|
||||
sample = self.last_sample
|
||||
return {
|
||||
"load": self.level.value,
|
||||
"loop_lag": round(sample.loop_lag, 4),
|
||||
"cpu": None if sample.cpu is None else round(sample.cpu, 3),
|
||||
"throttled": None if sample.throttled is None else round(sample.throttled, 3),
|
||||
}
|
||||
|
||||
def _measure(self, lag: float) -> LoadSample:
|
||||
now_cpu, now_wall = time.process_time(), time.monotonic()
|
||||
wall = now_wall - self._last_wall
|
||||
cpu = (now_cpu - self._last_cpu_time) / wall / self._cpu_capacity if wall > 0 else None
|
||||
self._last_cpu_time, self._last_wall = now_cpu, now_wall
|
||||
|
||||
throttled = None
|
||||
stats = self._cgroup_stats()
|
||||
if stats and self._last_cgroup:
|
||||
periods = stats.get("nr_periods", 0) - self._last_cgroup.get("nr_periods", 0)
|
||||
if periods > 0:
|
||||
throttled = (stats.get("nr_throttled", 0) - self._last_cgroup.get("nr_throttled", 0)) / periods
|
||||
self._last_cgroup = stats
|
||||
return LoadSample(loop_lag=lag, cpu=cpu, throttled=throttled)
|
||||
|
||||
def _cgroup_stats(self) -> dict[str, int]:
|
||||
text = _read(self._cgroup_dir / "cpu.stat")
|
||||
return parse_cpu_stat(text) if text else {}
|
||||
Reference in New Issue
Block a user