Add resource limits, abuse protection and overload handling

Running games always continue; new work is refused when there is no
capacity; abuse is rejected immediately.

- Docker: CPU/memory/pids caps per container (overridable in .env);
  uvicorn message size, queue and concurrency limits
- Server abuse limits: connections total/per IP (close 1013), message
  rate per connection (token bucket, close 1008 on floods), lobby
  creation rate per IP
- Load monitor: event loop lag, CPU of the own cgroup quota and
  throttling, with hysteresis (normal/busy/overloaded, severe lag
  overloads immediately)
- Admission control: create_lobby and start_game refused while
  overloaded or MAX_ACTIVE_GAMES are running; server_status message
  broadcast on changes; /health reports load but stays ok
- AI bridge: timeout sent with requests, in-flight cap, circuit breaker
- AI service: bounded workers and queue ("overloaded"), drops expired
  requests, degrades to the heuristic agent when busy or too slow
- Website: busy notices, Create/Start disabled when not accepting
  games, slow retry when the server is full
- Tests for limiters, load levels, admission control and AI service
  limits; docs updated

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

wizard_server/app/ai_bridge.py

@@ -4,14 +4,20 @@ The AI service (``wizard_ai``) connects to ``/internal/ai`` via websocket.
 Whenever an AI seat has to act, the server sends it the seat's observation
 and waits for an action:
 
-    server -> ai:  {"type": "decide", "request_id": "...", "observation": {...}}
-    ai -> server:  {"type": "action", "request_id": "...", "action": {...}}
+    server -> ai:  {"type": "decide", "request_id": "...", "timeout": 5.0, "observation": {...}}
+    ai -> server:  {"type": "action", "request_id": "...", "action": {...}, "agent": "heuristic"}
     ai -> server:  {"type": "error",  "request_id": "...", "message": "..."}
 
-Several AI service instances may connect; requests are distributed round
-robin. If no instance is connected, it does not answer in time, or it answers
-with an illegal action, a random legal action is played instead so a game
-never gets stuck.
+``timeout`` tells the AI service how long the server waits, so it can drop
+requests that are already too old. Several AI service instances may connect;
+requests are distributed round robin.
+
+A random legal action is played instead (so a game never gets stuck) when no
+instance is connected, it does not answer in time, it answers with an error or
+an illegal action, too many requests are already waiting for it
+(``max_inflight``), or the circuit breaker is open: after ``breaker_failures``
+failures in a row the AI is not asked for ``breaker_cooldown`` seconds, so a
+struggling AI service is not buried under even more requests.
 """
 
 from __future__ import annotations
@@ -55,6 +61,9 @@ class AIBridge:
         timeout: float,
         rng: random.Random | None = None,
         on_change: Callable[[], None] | None = None,
+        max_inflight: int = 32,
+        breaker_failures: int = 5,
+        breaker_cooldown: float = 30.0,
     ) -> None:
         self.timeout = timeout
         # Called whenever an AI service connects or disconnects.
@@ -62,6 +71,11 @@ class AIBridge:
         self._connections: list[AIConnection] = []
         self._round_robin = itertools.count()
         self._rng = rng or random.Random()
+        self._max_inflight = max_inflight
+        self._breaker_failures = breaker_failures
+        self._breaker_cooldown = breaker_cooldown
+        self._consecutive_failures = 0
+        self._breaker_open_until = 0.0
 
     @property
     def available(self) -> bool:
@@ -76,32 +90,49 @@ class AIBridge:
         connection = self._pick_connection()
         if connection is None:
             return self._fallback(observation, "no AI service connected", logging.DEBUG)
+        loop = asyncio.get_running_loop()
+        if loop.time() < self._breaker_open_until:
+            return self._fallback(observation, "AI paused after repeated failures", logging.DEBUG)
+        if len(connection.pending) >= self._max_inflight:
+            return self._failure(observation, f"{len(connection.pending)} AI requests already waiting")
 
         request_id = uuid.uuid4().hex
-        future: asyncio.Future[dict[str, Any]] = asyncio.get_running_loop().create_future()
+        future: asyncio.Future[dict[str, Any]] = loop.create_future()
         connection.pending[request_id] = future
         try:
             await connection.send({
                 "type": "decide",
                 "request_id": request_id,
+                "timeout": self.timeout,
                 "observation": observation.to_dict(),
             })
             reply = await asyncio.wait_for(future, self.timeout)
         except (TimeoutError, ConnectionError, RuntimeError, WebSocketDisconnect) as exc:
-            return self._fallback(observation, f"AI request failed: {exc!r}")
+            return self._failure(observation, f"AI request failed: {exc!r}")
         finally:
             connection.pending.pop(request_id, None)
 
         if reply.get("type") != "action":
-            return self._fallback(observation, f"AI error: {reply.get('message')}")
+            return self._failure(observation, f"AI error: {reply.get('message')}")
         try:
             action = action_from_dict(reply.get("action", {}))
         except ValueError as exc:
-            return self._fallback(observation, f"malformed AI action: {exc}")
+            return self._failure(observation, f"malformed AI action: {exc}")
         if action not in legal:
-            return self._fallback(observation, f"illegal AI action {action}")
+            return self._failure(observation, f"illegal AI action {action}")
+        self._consecutive_failures = 0
+        if reply.get("agent"):
+            logger.debug("AI decision by %s agent", reply["agent"])
         return action
 
+    def stats(self) -> dict[str, Any]:
+        loop_time = asyncio.get_running_loop().time()
+        return {
+            "connections": len(self._connections),
+            "pending": sum(len(c.pending) for c in self._connections),
+            "paused": loop_time < self._breaker_open_until,
+        }
+
     async def serve(self, websocket: WebSocket) -> None:
         """Handle one AI service connection until it disconnects."""
         connection = AIConnection(websocket, name=websocket.client.host if websocket.client else "?")
@@ -136,6 +167,16 @@ class AIBridge:
             return None
         return self._connections[next(self._round_robin) % len(self._connections)]
 
+    def _failure(self, observation: Observation, reason: str) -> Action:
+        """A failed AI request: count it for the circuit breaker, then fall back."""
+        self._consecutive_failures += 1
+        if self._consecutive_failures >= self._breaker_failures:
+            self._consecutive_failures = 0
+            self._breaker_open_until = asyncio.get_running_loop().time() + self._breaker_cooldown
+            logger.warning("AI failed %d times in a row, pausing AI requests for %.0fs",
+                           self._breaker_failures, self._breaker_cooldown)
+        return self._fallback(observation, reason)
+
     def _fallback(self, observation: Observation, reason: str, level: int = logging.WARNING) -> Action:
         logger.log(level, "Playing random move for player %d: %s", observation.player, reason)
         return self._rng.choice(observation.legal_actions)