Add resource limits, abuse protection and overload handling

Running games always continue; new work is refused when there is no
capacity; abuse is rejected immediately.

- Docker: CPU/memory/pids caps per container (overridable in .env);
  uvicorn message size, queue and concurrency limits
- Server abuse limits: connections total/per IP (close 1013), message
  rate per connection (token bucket, close 1008 on floods), lobby
  creation rate per IP
- Load monitor: event loop lag, CPU of the own cgroup quota and
  throttling, with hysteresis (normal/busy/overloaded, severe lag
  overloads immediately)
- Admission control: create_lobby and start_game refused while
  overloaded or MAX_ACTIVE_GAMES are running; server_status message
  broadcast on changes; /health reports load but stays ok
- AI bridge: timeout sent with requests, in-flight cap, circuit breaker
- AI service: bounded workers and queue ("overloaded"), drops expired
  requests, degrades to the heuristic agent when busy or too slow
- Website: busy notices, Create/Start disabled when not accepting
  games, slow retry when the server is full
- Tests for limiters, load levels, admission control and AI service
  limits; docs updated

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-09-27 12:51:46 +02:00
co-authored by Claude Opus 5.5
parent 905574e8f1
commit 87dfcd55b7
33 changed files with 1591 additions and 128 deletions
+40 -6
View File
@@ -5,7 +5,7 @@ from __future__ import annotations
import asyncio
import logging
import secrets
from collections.abc import Iterable
from collections.abc import Callable, Iterable
from dataclasses import dataclass
from enum import Enum
from typing import Any
@@ -24,6 +24,7 @@ from wizard_game import (
from .ai_bridge import AIBridge
from .config import Settings
from .protocol import ERROR_BUSY
from .sessions import Session
logger = logging.getLogger(__name__)
@@ -40,6 +41,11 @@ AI_NAMES = (
class LobbyError(Exception):
"""An error that is reported to the client that caused it."""
def __init__(self, message: str, code: str | None = None) -> None:
super().__init__(message)
# Optional machine readable code, see ``protocol.ERROR_*``.
self.code = code
class LobbyStatus(str, Enum):
WAITING = "waiting"
@@ -60,8 +66,16 @@ class Seat:
class Lobby:
def __init__(self, code: str, bridge: AIBridge, settings: Settings) -> None:
def __init__(
self,
code: str,
bridge: AIBridge,
settings: Settings,
on_status_change: Callable[[], None] | None = None,
) -> None:
self.code = code
# Called when a game starts or ends (the number of running games changes).
self._on_status_change = on_status_change
self.seats: list[Seat] = []
self.host_id: str | None = None
self.options = GameOptions()
@@ -188,6 +202,7 @@ class Lobby:
self.status = LobbyStatus.PLAYING
self._pause_until = 0.0
logger.info("Lobby %s: game started with %d players", self.code, len(self.seats))
self._notify_status_change()
self._schedule_ai()
def return_to_lobby(self, by: Session) -> None:
@@ -224,6 +239,9 @@ class Lobby:
self._closed = True
if self._ai_task and not self._ai_task.done():
self._ai_task.cancel()
if self.status is LobbyStatus.PLAYING:
self.status = LobbyStatus.FINISHED
self._notify_status_change()
# --- Internals ----------------------------------------------------------
@@ -247,12 +265,19 @@ class Lobby:
loop = asyncio.get_running_loop()
if any(isinstance(e, TrickCompleted) for e in events):
self._pause_until = loop.time() + self._settings.trick_pause
if any(isinstance(e, GameFinished) for e in events):
finished = any(isinstance(e, GameFinished) for e in events)
if finished:
self.status = LobbyStatus.FINISHED
logger.info("Lobby %s: game finished", self.code)
self.broadcast(events)
if finished:
self._notify_status_change()
self._schedule_ai()
def _notify_status_change(self) -> None:
if self._on_status_change is not None:
self._on_status_change()
def _current_ai_seat(self) -> int | None:
game = self.game
if self._closed or self.status is not LobbyStatus.PLAYING or game is None:
@@ -292,9 +317,15 @@ class Lobby:
class LobbyManager:
def __init__(self, bridge: AIBridge, settings: Settings) -> None:
def __init__(
self,
bridge: AIBridge,
settings: Settings,
on_games_changed: Callable[[], None] | None = None,
) -> None:
self._bridge = bridge
self._settings = settings
self._on_games_changed = on_games_changed
self._lobbies: dict[str, Lobby] = {}
def __len__(self) -> int:
@@ -303,14 +334,17 @@ class LobbyManager:
def __iter__(self):
return iter(list(self._lobbies.values()))
def active_games(self) -> int:
return sum(lobby.status is LobbyStatus.PLAYING for lobby in self._lobbies.values())
def create(self) -> Lobby:
if len(self._lobbies) >= self._settings.max_lobbies:
raise LobbyError("Too many open lobbies, please try again later")
raise LobbyError("Too many open lobbies, please try again later", ERROR_BUSY)
while True:
code = "".join(secrets.choice(LOBBY_CODE_ALPHABET) for _ in range(LOBBY_CODE_LENGTH))
if code not in self._lobbies:
break
lobby = Lobby(code, self._bridge, self._settings)
lobby = Lobby(code, self._bridge, self._settings, self._on_games_changed)
self._lobbies[code] = lobby
logger.info("Lobby %s created (%d open)", code, len(self._lobbies))
return lobby