Don't trip the AI circuit breaker on load shedding

With many simultaneous games, AI turns start in bursts and exceed the
in-flight cap. That was counted as an AI failure, so after 5 bursts the
breaker paused the AI for 30 s and every AI seat played random moves.

- Only real failures (timeouts, lost connection, malformed/illegal
  answers, unexpected errors) count towards the breaker; the in-flight
  cap and "overloaded"/"expired" answers from the AI service do not
- AI_MAX_INFLIGHT default 32 -> 64 (>= MAX_ACTIVE_GAMES, matches the AI
  service's AI_MAX_QUEUE)
- Test that load shedding keeps the breaker closed

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-09-27 13:04:21 +02:00
co-authored by Claude Opus 5.5
parent 87dfcd55b7
commit 763c5678f1
3 changed files with 58 additions and 7 deletions
+37 -1
View File
@@ -200,6 +200,40 @@ def test_ai_circuit_breaker():
asyncio.run(scenario())
class OverloadedAIWebSocket:
"""Answers every request immediately with the given error message."""
def __init__(self, bridge_connection_ref, message):
self.ref, self.message, self.sent = bridge_connection_ref, message, []
async def send_json(self, request):
self.sent.append(request)
future = self.ref[0].pending[request["request_id"]]
future.set_result({"type": "error", "request_id": request["request_id"], "message": self.message})
def test_load_shedding_does_not_trip_the_breaker():
async def scenario():
bridge = AIBridge(timeout=1, rng=random.Random(0), breaker_failures=2)
ref = []
ws = OverloadedAIWebSocket(ref, "overloaded")
connection = AIConnection(ws, "fake")
ref.append(connection)
bridge._connections.append(connection)
obs = observation()
for _ in range(5):
assert await bridge.decide(obs) in obs.legal_actions
assert len(ws.sent) == 5 # still asked every time
assert bridge.stats()["paused"] is False
ws.message = "model crashed" # a real failure counts
for _ in range(3):
await bridge.decide(obs)
assert len(ws.sent) == 7 and bridge.stats()["paused"] is True
asyncio.run(scenario())
def test_ai_inflight_limit():
async def scenario():
bridge = AIBridge(timeout=1, rng=random.Random(0), max_inflight=1)
@@ -208,7 +242,9 @@ def test_ai_inflight_limit():
connection.pending["busy"] = asyncio.get_running_loop().create_future()
bridge._connections.append(connection)
obs = observation()
assert await bridge.decide(obs) in obs.legal_actions
for _ in range(10):
assert await bridge.decide(obs) in obs.legal_actions
assert ws.sent == [] # not even sent: too many requests already waiting
assert bridge.stats()["paused"] is False # load shedding is not a failure
asyncio.run(scenario())