Add resource limits, abuse protection and overload handling

Running games always continue; new work is refused when there is no
capacity; abuse is rejected immediately.

- Docker: CPU/memory/pids caps per container (overridable in .env);
  uvicorn message size, queue and concurrency limits
- Server abuse limits: connections total/per IP (close 1013), message
  rate per connection (token bucket, close 1008 on floods), lobby
  creation rate per IP
- Load monitor: event loop lag, CPU of the own cgroup quota and
  throttling, with hysteresis (normal/busy/overloaded, severe lag
  overloads immediately)
- Admission control: create_lobby and start_game refused while
  overloaded or MAX_ACTIVE_GAMES are running; server_status message
  broadcast on changes; /health reports load but stays ok
- AI bridge: timeout sent with requests, in-flight cap, circuit breaker
- AI service: bounded workers and queue ("overloaded"), drops expired
  requests, degrades to the heuristic agent when busy or too slow
- Website: busy notices, Create/Start disabled when not accepting
  games, slow retry when the server is full
- Tests for limiters, load levels, admission control and AI service
  limits; docs updated

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

wizard_ai/tests/test_ai.py

@@ -78,3 +78,82 @@ def test_make_agent_falls_back_without_checkpoint(tmp_path):
 def test_heuristic_beats_random():
     result = evaluate([HeuristicAgent(), RandomAgent(1), RandomAgent(2), RandomAgent(3)], games=40, seed=0)
     assert result.mean_scores[0] > max(result.mean_scores[1:])
+
+
+# --- Service limits -------------------------------------------------------------
+
+import asyncio  # noqa: E402
+import threading  # noqa: E402
+
+from wizard_ai.service import AIService  # noqa: E402
+
+
+class BlockingAgent(RandomAgent):
+    """Blocks in act() until released, to simulate a slow model."""
+
+    name = "slow-model"
+
+    def __init__(self):
+        super().__init__(seed=0)
+        self.release = threading.Event()
+
+    def act(self, observation):
+        self.release.wait(5)
+        return super().act(observation)
+
+
+def decide_message(request_id, timeout=5.0):
+    game = WizardGame(3, seed=1)
+    return {"type": "decide", "request_id": request_id, "timeout": timeout,
+            "observation": game.observation(game.current_player).to_dict()}
+
+
+def test_service_answers_with_agent_name():
+    service = AIService("ws://unused", HeuristicAgent())
+    reply = asyncio.run(service.handle_request(decide_message("r1")))
+    assert reply["type"] == "action" and reply["request_id"] == "r1" and reply["agent"] == "heuristic"
+
+
+def test_service_refuses_when_queue_is_full_and_degrades_when_busy():
+    async def scenario():
+        slow = BlockingAgent()
+        service = AIService("ws://unused", slow, fallback=HeuristicAgent(), workers=1, max_queue=3, degrade_queue=1)
+        first = asyncio.create_task(service.handle_request(decide_message("a")))
+        await asyncio.sleep(0.05)  # "a" is computing with the slow agent
+        second = asyncio.create_task(service.handle_request(decide_message("b")))
+        third = asyncio.create_task(service.handle_request(decide_message("c")))
+        await asyncio.sleep(0.05)
+        refused = await service.handle_request(decide_message("d"))
+        assert refused == {"type": "error", "request_id": "d", "message": "overloaded"}
+        slow.release.set()
+        replies = await asyncio.gather(first, second, third)
+        # "b" starts while "c" is still waiting -> degraded; "c" runs when nothing waits anymore.
+        assert [r["agent"] for r in replies] == ["slow-model", "heuristic", "slow-model"]
+        assert service.in_progress == 0
+
+    asyncio.run(scenario())
+
+
+def test_service_drops_expired_requests():
+    async def scenario():
+        slow = BlockingAgent()
+        service = AIService("ws://unused", slow, workers=1)
+        first = asyncio.create_task(service.handle_request(decide_message("a")))
+        await asyncio.sleep(0.05)
+        waiting = asyncio.create_task(service.handle_request(decide_message("b", timeout=0.1)))
+        await asyncio.sleep(0.2)  # "b" expires while "a" blocks the only worker
+        slow.release.set()
+        await first
+        assert (await waiting)["message"] == "expired"
+
+    asyncio.run(scenario())
+
+
+def test_service_uses_fallback_when_primary_is_too_slow():
+    async def scenario():
+        service = AIService("ws://unused", RandomAgent(1), fallback=HeuristicAgent())
+        service._primary_latency = 3.0  # model needs 3 s, server waits 5 s -> more than half
+        reply = await service.handle_request(decide_message("a", timeout=5.0))
+        assert reply["agent"] == "heuristic"
+
+    asyncio.run(scenario())

wizard_ai/wizard_ai/__main__.py

@@ -49,10 +49,20 @@ def main() -> None:
     logging.basicConfig(level=args.log_level.upper(), format="%(asctime)s %(levelname)s %(name)s: %(message)s")
 
     if args.command == "serve":
+        from .agents import HeuristicAgent
         from .service import AIService
 
         agent = make_agent(args.agent, args.model_path)
-        asyncio.run(AIService(args.server_url, agent, config.secret).run_forever())
+        # Cheap agent used when the service is busy (pointless if it already is the heuristic).
+        fallback = None if isinstance(agent, HeuristicAgent) else HeuristicAgent()
+        service = AIService(
+            args.server_url, agent, config.secret,
+            fallback=fallback,
+            workers=config.workers,
+            max_queue=config.max_queue,
+            degrade_queue=config.degrade_queue,
+        )
+        asyncio.run(service.run_forever())
 
     elif args.command == "train":
         from .model import UniformPolicy, checkpoint_exists, load_policy

wizard_ai/wizard_ai/config.py

@@ -17,3 +17,9 @@ class Config:
     # Checkpoint directory of the trained policy.
     model_path: str = os.environ.get("MODEL_PATH", "models/latest")
     log_level: str = os.environ.get("LOG_LEVEL", "INFO")
+    # Decisions computed in parallel (agents must be thread safe if > 1).
+    workers: int = int(os.environ.get("AI_WORKERS", 1))
+    # Requests allowed to wait; more are refused with "overloaded".
+    max_queue: int = int(os.environ.get("AI_MAX_QUEUE", 64))
+    # Waiting requests from which the heuristic agent is used instead of the model.
+    degrade_queue: int = int(os.environ.get("AI_DEGRADE_QUEUE", 8))

wizard_ai/wizard_ai/service/client.py

@@ -3,6 +3,18 @@
 The service connects to the server's ``/internal/ai`` endpoint and answers
 ``decide`` requests (see ``wizard_server/app/ai_bridge.py``). It reconnects
 automatically if the connection drops.
+
+Work is bounded so a busy service degrades instead of falling over:
+
+- at most ``workers`` decisions are computed at the same time;
+- at most ``max_queue`` requests may wait; more are answered with the error
+  "overloaded" right away (the server then plays a fallback move);
+- a request that waited longer than the server's ``timeout`` is answered with
+  "expired" without computing it (the server has already moved on);
+- when ``degrade_queue`` or more other requests are waiting, or the primary
+  agent typically needs more than half of the remaining time, the cheap
+  ``fallback`` agent (heuristic) decides instead of the primary agent (e.g. a
+  neural network).
 """
 
 from __future__ import annotations
@@ -10,6 +22,8 @@ from __future__ import annotations
 import asyncio
 import json
 import logging
+import time
+from collections.abc import Callable
 from typing import Any
 
 from websockets.asyncio.client import ClientConnection, connect
@@ -22,16 +36,36 @@ from ..agents import Agent
 logger = logging.getLogger(__name__)
 
 MAX_BACKOFF = 30.0
+# Weight of the newest measurement in the average decision time.
+LATENCY_SMOOTHING = 0.2
 
 
 class AIService:
-    def __init__(self, server_url: str, agent: Agent, secret: str = "") -> None:
+    def __init__(
+        self,
+        server_url: str,
+        agent: Agent,
+        secret: str = "",
+        *,
+        fallback: Agent | None = None,
+        workers: int = 1,
+        max_queue: int = 64,
+        degrade_queue: int = 8,
+        clock: Callable[[], float] = time.monotonic,
+    ) -> None:
         self.server_url = server_url
         self.agent = agent
+        self.fallback = fallback
         self.secret = secret
-        # Inference runs in worker threads; the lock keeps agents that are not
-        # thread safe (e.g. stateful models) consistent.
-        self._agent_lock = asyncio.Lock()
+        self.max_queue = max_queue
+        self.degrade_queue = degrade_queue
+        self._clock = clock
+        # Agents run in worker threads; with workers > 1 they must be thread safe.
+        self._workers = asyncio.Semaphore(max(1, workers))
+        # Requests received and not answered yet (waiting + computing).
+        self.in_progress = 0
+        # Smoothed seconds the primary agent needs per decision.
+        self._primary_latency = 0.0
 
     async def run_forever(self) -> None:
         backoff = 1.0
@@ -55,21 +89,62 @@ class AIService:
             async for raw in websocket:
                 message = json.loads(raw)
                 if message.get("type") == "decide":
-                    task = asyncio.create_task(self._decide(websocket, message))
+                    task = asyncio.create_task(self._answer(websocket, message))
                     tasks.add(task)
                     task.add_done_callback(tasks.discard)
 
-    async def _decide(self, websocket: ClientConnection, message: dict[str, Any]) -> None:
-        request_id = message.get("request_id")
-        try:
-            observation = Observation.from_dict(message["observation"])
-            async with self._agent_lock:
-                action = await asyncio.to_thread(self.agent.act, observation)
-            reply = {"type": "action", "request_id": request_id, "action": action.to_dict()}
-        except Exception as exc:  # report instead of crashing; the server falls back
-            logger.exception("Failed to decide request %s", request_id)
-            reply = {"type": "error", "request_id": request_id, "message": str(exc)}
+    async def _answer(self, websocket: ClientConnection, message: dict[str, Any]) -> None:
+        reply = await self.handle_request(message)
         try:
             await websocket.send(json.dumps(reply))
         except ConnectionClosed:
             pass
+
+    async def handle_request(self, message: dict[str, Any]) -> dict[str, Any]:
+        """Answer one ``decide`` request (independent of the websocket, for tests)."""
+        request_id = message.get("request_id")
+        received = self._clock()
+        timeout = message.get("timeout")
+        deadline = received + timeout if isinstance(timeout, (int, float)) and timeout > 0 else None
+
+        if self.in_progress >= self.max_queue:
+            logger.warning("Overloaded: %d requests in progress, refusing %s", self.in_progress, request_id)
+            return self._error(request_id, "overloaded")
+
+        self.in_progress += 1
+        try:
+            async with self._workers:
+                now = self._clock()
+                if deadline is not None and now >= deadline:
+                    logger.info("Dropping expired request %s (waited %.2fs)", request_id, now - received)
+                    return self._error(request_id, "expired")
+                agent = self._choose_agent(deadline, now)
+                observation = Observation.from_dict(message["observation"])
+                started = self._clock()
+                action = await asyncio.to_thread(agent.act, observation)
+                if agent is self.agent:
+                    elapsed = self._clock() - started
+                    self._primary_latency += LATENCY_SMOOTHING * (elapsed - self._primary_latency)
+            return {"type": "action", "request_id": request_id, "action": action.to_dict(), "agent": agent.name}
+        except Exception as exc:  # report instead of crashing; the server falls back
+            logger.exception("Failed to decide request %s", request_id)
+            return self._error(request_id, str(exc) or type(exc).__name__)
+        finally:
+            self.in_progress -= 1
+
+    def _choose_agent(self, deadline: float | None, now: float) -> Agent:
+        """The primary agent, or the cheap fallback when the service is busy."""
+        if self.fallback is None:
+            return self.agent
+        waiting = self.in_progress - 1  # requests besides this one
+        if waiting >= self.degrade_queue:
+            logger.info("Busy (%d waiting): using %s agent", waiting, self.fallback.name)
+            return self.fallback
+        if deadline is not None and self._primary_latency > (deadline - now) / 2:
+            logger.info("Slow (%.2fs per decision): using %s agent", self._primary_latency, self.fallback.name)
+            return self.fallback
+        return self.agent
+
+    @staticmethod
+    def _error(request_id: Any, message: str) -> dict[str, Any]:
+        return {"type": "error", "request_id": request_id, "message": message}