Don't trip the AI circuit breaker on load shedding

With many simultaneous games, AI turns start in bursts and exceed the
in-flight cap. That was counted as an AI failure, so after 5 bursts the
breaker paused the AI for 30 s and every AI seat played random moves.

- Only real failures (timeouts, lost connection, malformed/illegal
  answers, unexpected errors) count towards the breaker; the in-flight
  cap and "overloaded"/"expired" answers from the AI service do not
- AI_MAX_INFLIGHT default 32 -> 64 (>= MAX_ACTIVE_GAMES, matches the AI
  service's AI_MAX_QUEUE)
- Test that load shedding keeps the breaker closed

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

wizard_server/app/ai_bridge.py

@@ -15,9 +15,15 @@ requests are distributed round robin.
 A random legal action is played instead (so a game never gets stuck) when no
 instance is connected, it does not answer in time, it answers with an error or
 an illegal action, too many requests are already waiting for it
-(``max_inflight``), or the circuit breaker is open: after ``breaker_failures``
-failures in a row the AI is not asked for ``breaker_cooldown`` seconds, so a
-struggling AI service is not buried under even more requests.
+(``max_inflight``), or the circuit breaker is open.
+
+The circuit breaker protects against a *broken or hanging* AI service: after
+``breaker_failures`` real failures in a row (timeouts, lost connection,
+malformed or illegal answers, unexpected errors) the AI is not asked for
+``breaker_cooldown`` seconds, so games don't wait ``timeout`` seconds for every
+AI move. Deliberate load shedding is *not* a failure and does not trip it: the
+in-flight cap, and the AI answering "overloaded" or "expired" (it is healthy
+and answered immediately).
 """
 
 from __future__ import annotations
@@ -36,6 +42,9 @@ from wizard_game import Action, Observation, action_from_dict
 
 logger = logging.getLogger(__name__)
 
+# Errors with which a healthy AI service sheds load (see wizard_ai/service/client.py).
+LOAD_SHEDDING_ERRORS = ("overloaded", "expired")
+
 
 class AIConnection:
     def __init__(self, websocket: WebSocket, name: str) -> None:
@@ -94,7 +103,7 @@ class AIBridge:
         if loop.time() < self._breaker_open_until:
             return self._fallback(observation, "AI paused after repeated failures", logging.DEBUG)
         if len(connection.pending) >= self._max_inflight:
-            return self._failure(observation, f"{len(connection.pending)} AI requests already waiting")
+            return self._fallback(observation, f"{len(connection.pending)} AI requests already waiting")
 
         request_id = uuid.uuid4().hex
         future: asyncio.Future[dict[str, Any]] = loop.create_future()
@@ -113,7 +122,11 @@ class AIBridge:
             connection.pending.pop(request_id, None)
 
         if reply.get("type") != "action":
-            return self._failure(observation, f"AI error: {reply.get('message')}")
+            message = reply.get("message")
+            if message in LOAD_SHEDDING_ERRORS:
+                self._consecutive_failures = 0  # the service is alive and protecting itself
+                return self._fallback(observation, f"AI service {message}")
+            return self._failure(observation, f"AI error: {message}")
         try:
             action = action_from_dict(reply.get("action", {}))
         except ValueError as exc:

wizard_server/app/config.py

@@ -69,7 +69,9 @@ class Settings:
 
     # --- AI service protection ----------------------------------------------
     # Unanswered decide requests per AI connection before falling back.
-    ai_max_inflight: int = _int("AI_MAX_INFLIGHT", 32)
+    # Each running game has at most one AI deciding at a time, so keep this at
+    # least MAX_ACTIVE_GAMES; 64 matches the AI service's AI_MAX_QUEUE.
+    ai_max_inflight: int = _int("AI_MAX_INFLIGHT", 64)
     # Consecutive AI failures that pause AI requests for ai_breaker_cooldown seconds.
     ai_breaker_failures: int = _int("AI_BREAKER_FAILURES", 5)
     ai_breaker_cooldown: float = _float("AI_BREAKER_COOLDOWN", 30.0)

wizard_server/tests/test_limits.py

@@ -200,6 +200,40 @@ def test_ai_circuit_breaker():
     asyncio.run(scenario())
 
 
+class OverloadedAIWebSocket:
+    """Answers every request immediately with the given error message."""
+
+    def __init__(self, bridge_connection_ref, message):
+        self.ref, self.message, self.sent = bridge_connection_ref, message, []
+
+    async def send_json(self, request):
+        self.sent.append(request)
+        future = self.ref[0].pending[request["request_id"]]
+        future.set_result({"type": "error", "request_id": request["request_id"], "message": self.message})
+
+
+def test_load_shedding_does_not_trip_the_breaker():
+    async def scenario():
+        bridge = AIBridge(timeout=1, rng=random.Random(0), breaker_failures=2)
+        ref = []
+        ws = OverloadedAIWebSocket(ref, "overloaded")
+        connection = AIConnection(ws, "fake")
+        ref.append(connection)
+        bridge._connections.append(connection)
+        obs = observation()
+        for _ in range(5):
+            assert await bridge.decide(obs) in obs.legal_actions
+        assert len(ws.sent) == 5  # still asked every time
+        assert bridge.stats()["paused"] is False
+
+        ws.message = "model crashed"  # a real failure counts
+        for _ in range(3):
+            await bridge.decide(obs)
+        assert len(ws.sent) == 7 and bridge.stats()["paused"] is True
+
+    asyncio.run(scenario())
+
+
 def test_ai_inflight_limit():
     async def scenario():
         bridge = AIBridge(timeout=1, rng=random.Random(0), max_inflight=1)
@@ -208,7 +242,9 @@ def test_ai_inflight_limit():
         connection.pending["busy"] = asyncio.get_running_loop().create_future()
         bridge._connections.append(connection)
         obs = observation()
-        assert await bridge.decide(obs) in obs.legal_actions
+        for _ in range(10):
+            assert await bridge.decide(obs) in obs.legal_actions
         assert ws.sent == []  # not even sent: too many requests already waiting
+        assert bridge.stats()["paused"] is False  # load shedding is not a failure
 
     asyncio.run(scenario())