diff --git a/wizard_server/app/ai_bridge.py b/wizard_server/app/ai_bridge.py index e66a47b..d797501 100644 --- a/wizard_server/app/ai_bridge.py +++ b/wizard_server/app/ai_bridge.py @@ -15,9 +15,15 @@ requests are distributed round robin. A random legal action is played instead (so a game never gets stuck) when no instance is connected, it does not answer in time, it answers with an error or an illegal action, too many requests are already waiting for it -(``max_inflight``), or the circuit breaker is open: after ``breaker_failures`` -failures in a row the AI is not asked for ``breaker_cooldown`` seconds, so a -struggling AI service is not buried under even more requests. +(``max_inflight``), or the circuit breaker is open. + +The circuit breaker protects against a *broken or hanging* AI service: after +``breaker_failures`` real failures in a row (timeouts, lost connection, +malformed or illegal answers, unexpected errors) the AI is not asked for +``breaker_cooldown`` seconds, so games don't wait ``timeout`` seconds for every +AI move. Deliberate load shedding is *not* a failure and does not trip it: the +in-flight cap, and the AI answering "overloaded" or "expired" (it is healthy +and answered immediately). """ from __future__ import annotations @@ -36,6 +42,9 @@ from wizard_game import Action, Observation, action_from_dict logger = logging.getLogger(__name__) +# Errors with which a healthy AI service sheds load (see wizard_ai/service/client.py). +LOAD_SHEDDING_ERRORS = ("overloaded", "expired") + class AIConnection: def __init__(self, websocket: WebSocket, name: str) -> None: @@ -94,7 +103,7 @@ class AIBridge: if loop.time() < self._breaker_open_until: return self._fallback(observation, "AI paused after repeated failures", logging.DEBUG) if len(connection.pending) >= self._max_inflight: - return self._failure(observation, f"{len(connection.pending)} AI requests already waiting") + return self._fallback(observation, f"{len(connection.pending)} AI requests already waiting") request_id = uuid.uuid4().hex future: asyncio.Future[dict[str, Any]] = loop.create_future() @@ -113,7 +122,11 @@ class AIBridge: connection.pending.pop(request_id, None) if reply.get("type") != "action": - return self._failure(observation, f"AI error: {reply.get('message')}") + message = reply.get("message") + if message in LOAD_SHEDDING_ERRORS: + self._consecutive_failures = 0 # the service is alive and protecting itself + return self._fallback(observation, f"AI service {message}") + return self._failure(observation, f"AI error: {message}") try: action = action_from_dict(reply.get("action", {})) except ValueError as exc: diff --git a/wizard_server/app/config.py b/wizard_server/app/config.py index 9bd1de8..4d9acc8 100644 --- a/wizard_server/app/config.py +++ b/wizard_server/app/config.py @@ -69,7 +69,9 @@ class Settings: # --- AI service protection ---------------------------------------------- # Unanswered decide requests per AI connection before falling back. - ai_max_inflight: int = _int("AI_MAX_INFLIGHT", 32) + # Each running game has at most one AI deciding at a time, so keep this at + # least MAX_ACTIVE_GAMES; 64 matches the AI service's AI_MAX_QUEUE. + ai_max_inflight: int = _int("AI_MAX_INFLIGHT", 64) # Consecutive AI failures that pause AI requests for ai_breaker_cooldown seconds. ai_breaker_failures: int = _int("AI_BREAKER_FAILURES", 5) ai_breaker_cooldown: float = _float("AI_BREAKER_COOLDOWN", 30.0) diff --git a/wizard_server/tests/test_limits.py b/wizard_server/tests/test_limits.py index de21235..4a6c2c1 100644 --- a/wizard_server/tests/test_limits.py +++ b/wizard_server/tests/test_limits.py @@ -200,6 +200,40 @@ def test_ai_circuit_breaker(): asyncio.run(scenario()) +class OverloadedAIWebSocket: + """Answers every request immediately with the given error message.""" + + def __init__(self, bridge_connection_ref, message): + self.ref, self.message, self.sent = bridge_connection_ref, message, [] + + async def send_json(self, request): + self.sent.append(request) + future = self.ref[0].pending[request["request_id"]] + future.set_result({"type": "error", "request_id": request["request_id"], "message": self.message}) + + +def test_load_shedding_does_not_trip_the_breaker(): + async def scenario(): + bridge = AIBridge(timeout=1, rng=random.Random(0), breaker_failures=2) + ref = [] + ws = OverloadedAIWebSocket(ref, "overloaded") + connection = AIConnection(ws, "fake") + ref.append(connection) + bridge._connections.append(connection) + obs = observation() + for _ in range(5): + assert await bridge.decide(obs) in obs.legal_actions + assert len(ws.sent) == 5 # still asked every time + assert bridge.stats()["paused"] is False + + ws.message = "model crashed" # a real failure counts + for _ in range(3): + await bridge.decide(obs) + assert len(ws.sent) == 7 and bridge.stats()["paused"] is True + + asyncio.run(scenario()) + + def test_ai_inflight_limit(): async def scenario(): bridge = AIBridge(timeout=1, rng=random.Random(0), max_inflight=1) @@ -208,7 +242,9 @@ def test_ai_inflight_limit(): connection.pending["busy"] = asyncio.get_running_loop().create_future() bridge._connections.append(connection) obs = observation() - assert await bridge.decide(obs) in obs.legal_actions + for _ in range(10): + assert await bridge.decide(obs) in obs.legal_actions assert ws.sent == [] # not even sent: too many requests already waiting + assert bridge.stats()["paused"] is False # load shedding is not a failure asyncio.run(scenario())