Add resource limits, abuse protection and overload handling

Running games always continue; new work is refused when there is no
capacity; abuse is rejected immediately.

- Docker: CPU/memory/pids caps per container (overridable in .env);
  uvicorn message size, queue and concurrency limits
- Server abuse limits: connections total/per IP (close 1013), message
  rate per connection (token bucket, close 1008 on floods), lobby
  creation rate per IP
- Load monitor: event loop lag, CPU of the own cgroup quota and
  throttling, with hysteresis (normal/busy/overloaded, severe lag
  overloads immediately)
- Admission control: create_lobby and start_game refused while
  overloaded or MAX_ACTIVE_GAMES are running; server_status message
  broadcast on changes; /health reports load but stays ok
- AI bridge: timeout sent with requests, in-flight cap, circuit breaker
- AI service: bounded workers and queue ("overloaded"), drops expired
  requests, degrades to the heuristic agent when busy or too slow
- Website: busy notices, Create/Start disabled when not accepting
  games, slow retry when the server is full
- Tests for limiters, load levels, admission control and AI service
  limits; docs updated

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-09-27 12:51:46 +02:00
co-authored by Claude Opus 5.5
parent 905574e8f1
commit 87dfcd55b7
33 changed files with 1591 additions and 128 deletions
+90
View File
@@ -0,0 +1,90 @@
"""Rate and connection limits that protect the server from floods and abuse."""
from __future__ import annotations
import time
from collections import Counter
from collections.abc import Callable, Hashable
Clock = Callable[[], float]
class TokenBucket:
"""Allows ``rate`` actions per second on average with bursts up to ``burst``."""
def __init__(self, rate: float, burst: float, clock: Clock = time.monotonic) -> None:
self.rate = rate
self.burst = burst
self._clock = clock
self._tokens = float(burst)
self._updated = clock()
def allow(self, cost: float = 1.0) -> bool:
self._refill()
if self._tokens >= cost:
self._tokens -= cost
return True
return False
@property
def full(self) -> bool:
self._refill()
return self._tokens >= self.burst
def _refill(self) -> None:
now = self._clock()
self._tokens = min(self.burst, self._tokens + (now - self._updated) * self.rate)
self._updated = now
class KeyedRateLimiter:
"""One token bucket per key (e.g. per client IP)."""
def __init__(self, rate: float, burst: float, clock: Clock = time.monotonic) -> None:
self.rate = rate
self.burst = burst
self._clock = clock
self._buckets: dict[Hashable, TokenBucket] = {}
def allow(self, key: Hashable) -> bool:
bucket = self._buckets.get(key)
if bucket is None:
bucket = self._buckets[key] = TokenBucket(self.rate, self.burst, self._clock)
return bucket.allow()
def prune(self) -> None:
"""Forget keys whose bucket has refilled completely (they behave like new keys)."""
for key in [key for key, bucket in self._buckets.items() if bucket.full]:
del self._buckets[key]
def __len__(self) -> int:
return len(self._buckets)
class ConnectionLimiter:
"""Counts open connections in total and per client IP."""
def __init__(self, max_total: int, max_per_ip: int) -> None:
self.max_total = max_total
self.max_per_ip = max_per_ip
self._per_ip: Counter[str] = Counter()
self._total = 0
@property
def total(self) -> int:
return self._total
def acquire(self, ip: str) -> bool:
if self._total >= self.max_total or self._per_ip[ip] >= self.max_per_ip:
return False
self._per_ip[ip] += 1
self._total += 1
return True
def release(self, ip: str) -> None:
if self._per_ip[ip] <= 0:
return
self._per_ip[ip] -= 1
self._total -= 1
if self._per_ip[ip] == 0:
del self._per_ip[ip]