Add resource limits, abuse protection and overload handling
Running games always continue; new work is refused when there is no
capacity; abuse is rejected immediately.
- Docker: CPU/memory/pids caps per container (overridable in .env);
uvicorn message size, queue and concurrency limits
- Server abuse limits: connections total/per IP (close 1013), message
rate per connection (token bucket, close 1008 on floods), lobby
creation rate per IP
- Load monitor: event loop lag, CPU of the own cgroup quota and
throttling, with hysteresis (normal/busy/overloaded, severe lag
overloads immediately)
- Admission control: create_lobby and start_game refused while
overloaded or MAX_ACTIVE_GAMES are running; server_status message
broadcast on changes; /health reports load but stays ok
- AI bridge: timeout sent with requests, in-flight cap, circuit breaker
- AI service: bounded workers and queue ("overloaded"), drops expired
requests, degrades to the heuristic agent when busy or too slow
- Website: busy notices, Create/Start disabled when not accepting
games, slow retry when the server is full
- Tests for limiters, load levels, admission control and AI service
limits; docs updated
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -5,7 +5,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import logging
|
||||
import secrets
|
||||
from collections.abc import Iterable
|
||||
from collections.abc import Callable, Iterable
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
@@ -24,6 +24,7 @@ from wizard_game import (
|
||||
|
||||
from .ai_bridge import AIBridge
|
||||
from .config import Settings
|
||||
from .protocol import ERROR_BUSY
|
||||
from .sessions import Session
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -40,6 +41,11 @@ AI_NAMES = (
|
||||
class LobbyError(Exception):
|
||||
"""An error that is reported to the client that caused it."""
|
||||
|
||||
def __init__(self, message: str, code: str | None = None) -> None:
|
||||
super().__init__(message)
|
||||
# Optional machine readable code, see ``protocol.ERROR_*``.
|
||||
self.code = code
|
||||
|
||||
|
||||
class LobbyStatus(str, Enum):
|
||||
WAITING = "waiting"
|
||||
@@ -60,8 +66,16 @@ class Seat:
|
||||
|
||||
|
||||
class Lobby:
|
||||
def __init__(self, code: str, bridge: AIBridge, settings: Settings) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
code: str,
|
||||
bridge: AIBridge,
|
||||
settings: Settings,
|
||||
on_status_change: Callable[[], None] | None = None,
|
||||
) -> None:
|
||||
self.code = code
|
||||
# Called when a game starts or ends (the number of running games changes).
|
||||
self._on_status_change = on_status_change
|
||||
self.seats: list[Seat] = []
|
||||
self.host_id: str | None = None
|
||||
self.options = GameOptions()
|
||||
@@ -188,6 +202,7 @@ class Lobby:
|
||||
self.status = LobbyStatus.PLAYING
|
||||
self._pause_until = 0.0
|
||||
logger.info("Lobby %s: game started with %d players", self.code, len(self.seats))
|
||||
self._notify_status_change()
|
||||
self._schedule_ai()
|
||||
|
||||
def return_to_lobby(self, by: Session) -> None:
|
||||
@@ -224,6 +239,9 @@ class Lobby:
|
||||
self._closed = True
|
||||
if self._ai_task and not self._ai_task.done():
|
||||
self._ai_task.cancel()
|
||||
if self.status is LobbyStatus.PLAYING:
|
||||
self.status = LobbyStatus.FINISHED
|
||||
self._notify_status_change()
|
||||
|
||||
# --- Internals ----------------------------------------------------------
|
||||
|
||||
@@ -247,12 +265,19 @@ class Lobby:
|
||||
loop = asyncio.get_running_loop()
|
||||
if any(isinstance(e, TrickCompleted) for e in events):
|
||||
self._pause_until = loop.time() + self._settings.trick_pause
|
||||
if any(isinstance(e, GameFinished) for e in events):
|
||||
finished = any(isinstance(e, GameFinished) for e in events)
|
||||
if finished:
|
||||
self.status = LobbyStatus.FINISHED
|
||||
logger.info("Lobby %s: game finished", self.code)
|
||||
self.broadcast(events)
|
||||
if finished:
|
||||
self._notify_status_change()
|
||||
self._schedule_ai()
|
||||
|
||||
def _notify_status_change(self) -> None:
|
||||
if self._on_status_change is not None:
|
||||
self._on_status_change()
|
||||
|
||||
def _current_ai_seat(self) -> int | None:
|
||||
game = self.game
|
||||
if self._closed or self.status is not LobbyStatus.PLAYING or game is None:
|
||||
@@ -292,9 +317,15 @@ class Lobby:
|
||||
|
||||
|
||||
class LobbyManager:
|
||||
def __init__(self, bridge: AIBridge, settings: Settings) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
bridge: AIBridge,
|
||||
settings: Settings,
|
||||
on_games_changed: Callable[[], None] | None = None,
|
||||
) -> None:
|
||||
self._bridge = bridge
|
||||
self._settings = settings
|
||||
self._on_games_changed = on_games_changed
|
||||
self._lobbies: dict[str, Lobby] = {}
|
||||
|
||||
def __len__(self) -> int:
|
||||
@@ -303,14 +334,17 @@ class LobbyManager:
|
||||
def __iter__(self):
|
||||
return iter(list(self._lobbies.values()))
|
||||
|
||||
def active_games(self) -> int:
|
||||
return sum(lobby.status is LobbyStatus.PLAYING for lobby in self._lobbies.values())
|
||||
|
||||
def create(self) -> Lobby:
|
||||
if len(self._lobbies) >= self._settings.max_lobbies:
|
||||
raise LobbyError("Too many open lobbies, please try again later")
|
||||
raise LobbyError("Too many open lobbies, please try again later", ERROR_BUSY)
|
||||
while True:
|
||||
code = "".join(secrets.choice(LOBBY_CODE_ALPHABET) for _ in range(LOBBY_CODE_LENGTH))
|
||||
if code not in self._lobbies:
|
||||
break
|
||||
lobby = Lobby(code, self._bridge, self._settings)
|
||||
lobby = Lobby(code, self._bridge, self._settings, self._on_games_changed)
|
||||
self._lobbies[code] = lobby
|
||||
logger.info("Lobby %s created (%d open)", code, len(self._lobbies))
|
||||
return lobby
|
||||
|
||||
Reference in New Issue
Block a user