marvin/Wizard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add resource limits, abuse protection and overload handling

Browse Source 
Running games always continue; new work is refused when there is no
capacity; abuse is rejected immediately.

- Docker: CPU/memory/pids caps per container (overridable in .env);
  uvicorn message size, queue and concurrency limits
- Server abuse limits: connections total/per IP (close 1013), message
  rate per connection (token bucket, close 1008 on floods), lobby
  creation rate per IP
- Load monitor: event loop lag, CPU of the own cgroup quota and
  throttling, with hysteresis (normal/busy/overloaded, severe lag
  overloads immediately)
- Admission control: create_lobby and start_game refused while
  overloaded or MAX_ACTIVE_GAMES are running; server_status message
  broadcast on changes; /health reports load but stays ok
- AI bridge: timeout sent with requests, in-flight cap, circuit breaker
- AI service: bounded workers and queue ("overloaded"), drops expired
  requests, degrades to the heuristic agent when busy or too slow
- Website: busy notices, Create/Start disabled when not accepting
  games, slow retry when the server is full
- Tests for limiters, load levels, admission control and AI service
  limits; docs updated

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
marvin
2026-09-27 12:51:46 +02:00
co-authored by Claude Opus 5.5
parent 905574e8f1
commit 87dfcd55b7
33 changed files with 1591 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
wizard_server/app/service.py
+89 -4
View File
@@ -15,6 +15,8 @@ from wizard_game import GameOptions
from . import protocol
from .ai_bridge import AIBridge
from .config import Settings
from .limits import ConnectionLimiter, KeyedRateLimiter, TokenBucket
from .load import LoadLevel, LoadMonitor
from .lobby import Lobby, LobbyError, LobbyManager
from .sessions import Connection, Session, SessionRegistry
@@ -27,8 +29,53 @@ class WizardService:
def __init__(self, settings: Settings) -> None:
self.settings = settings
self.sessions = SessionRegistry()
self.bridge = AIBridge(settings.ai_decision_timeout, on_change=self._on_ai_availability_changed)
self.lobbies = LobbyManager(self.bridge, settings)
self.bridge = AIBridge(
settings.ai_decision_timeout,
on_change=self._on_ai_availability_changed,
max_inflight=settings.ai_max_inflight,
breaker_failures=settings.ai_breaker_failures,
breaker_cooldown=settings.ai_breaker_cooldown,
)
self.lobbies = LobbyManager(self.bridge, settings, on_games_changed=self._broadcast_server_status)
self.connections = ConnectionLimiter(settings.max_connections, settings.max_connections_per_ip)
self.lobby_creation = KeyedRateLimiter(settings.lobby_create_rate / 60, settings.lobby_create_rate)
self.load = LoadMonitor(settings, on_change=lambda _level: self._broadcast_server_status())
self._last_server_status: dict | None = None
# --- Capacity -----------------------------------------------------------
@property
def accepting_games(self) -> bool:
"""Whether new lobbies may be created and games started.
Running games are never affected; only new work is refused.
"""
return (
self.load.level is not LoadLevel.OVERLOADED
and self.lobbies.active_games() < self.settings.max_active_games
)
def server_status(self) -> dict:
return {
"type": "server_status",
"load": self.load.level.value,
"accepting_games": self.accepting_games,
}
def _broadcast_server_status(self) -> None:
status = self.server_status()
if status == self._last_server_status:
return
self._last_server_status = status
for session in self.sessions.connected():
session.send(status)
def _require_capacity(self) -> None:
if not self.accepting_games:
raise LobbyError(
"The server is busy right now, new games can't be started. Please try again in a moment.",
protocol.ERROR_BUSY,
)
# --- Client connections -------------------------------------------------
@@ -41,10 +88,26 @@ class WizardService:
await websocket.accept()
connection = Connection(websocket)
connection.start()
if not self.connections.acquire(connection.ip):
logger.warning("Refusing connection from %s: connection limit reached", connection.ip)
connection.send(protocol.error("The server is full right now, please try again later.", protocol.ERROR_BUSY))
await connection.close(code=protocol.CLOSE_TRY_AGAIN_LATER)
return
bucket = TokenBucket(self.settings.message_rate, self.settings.message_burst)
violations = 0
session: Session | None = None
try:
while True:
raw = await websocket.receive_text()
if not bucket.allow():
violations += 1
if violations >= self.settings.max_rate_violations:
logger.warning("Closing connection from %s: message flood", connection.ip)
break
if violations == 1 or violations % 10 == 0:
connection.send(protocol.error("Too many messages, slow down.", protocol.ERROR_RATE_LIMITED))
continue
try:
message = protocol.client_message_adapter.validate_json(raw)
except ValidationError as exc:
@@ -60,16 +123,17 @@ class WizardService:
try:
self._dispatch(session, message)
except LobbyError as exc:
session.send(protocol.error(str(exc)))
session.send(protocol.error(str(exc), exc.code))
except WebSocketDisconnect:
pass
finally:
self.connections.release(connection.ip)
if session is not None and session.connection is connection:
session.connection = None
session.last_seen = time.monotonic()
if lobby := self.lobbies.get(session.lobby_code):
lobby.broadcast()
await connection.close()
await connection.close(code=1008 if violations >= self.settings.max_rate_violations else 1000)
def _hello(self, connection: Connection, token: str | None) -> Session:
session = self.sessions.get(token)
@@ -87,6 +151,7 @@ class WizardService:
"token": session.token,
"name": session.name,
})
session.send(self.server_status())
lobby = self.lobbies.get(session.lobby_code)
if lobby is not None and lobby.seat_index(session.player_id) is not None:
lobby.add_human(session) # re-binds the seat to this session
@@ -99,6 +164,11 @@ class WizardService:
def _dispatch(self, session: Session, message: object) -> None:
match message:
case protocol.CreateLobby(name=name):
self._require_capacity()
ip = session.connection.ip if session.connection else "unknown"
if not self.lobby_creation.allow(ip):
raise LobbyError("You are creating lobbies too quickly, please wait a moment.",
protocol.ERROR_RATE_LIMITED)
self._leave_current_lobby(session)
session.name = name
lobby = self.lobbies.create()
@@ -132,6 +202,7 @@ class WizardService:
lobby.broadcast()
case protocol.StartGame():
lobby = self._lobby_of(session)
self._require_capacity()
lobby.start(session)
lobby.broadcast()
case protocol.ReturnToLobby():
@@ -200,6 +271,20 @@ class WizardService:
for session in self.sessions.expired(self.settings.session_ttl):
self._leave_current_lobby(session)
self.sessions.remove(session)
self.lobby_creation.prune()
def health(self) -> dict:
return {
"status": "ok", # stays ok under load: an overloaded server must not be restarted
**self.load.stats(),
"accepting_games": self.accepting_games,
"lobbies": len(self.lobbies),
"active_games": self.lobbies.active_games(),
"sessions": len(self.sessions),
"connections": self.connections.total,
"ai_available": self.bridge.available,
"ai": self.bridge.stats(),
}
def _validation_message(exc: ValidationError) -> str: