Don't trip the AI circuit breaker on load shedding
With many simultaneous games, AI turns start in bursts and exceed the in-flight cap. That was counted as an AI failure, so after 5 bursts the breaker paused the AI for 30 s and every AI seat played random moves. - Only real failures (timeouts, lost connection, malformed/illegal answers, unexpected errors) count towards the breaker; the in-flight cap and "overloaded"/"expired" answers from the AI service do not - AI_MAX_INFLIGHT default 32 -> 64 (>= MAX_ACTIVE_GAMES, matches the AI service's AI_MAX_QUEUE) - Test that load shedding keeps the breaker closed Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -200,6 +200,40 @@ def test_ai_circuit_breaker():
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
class OverloadedAIWebSocket:
|
||||
"""Answers every request immediately with the given error message."""
|
||||
|
||||
def __init__(self, bridge_connection_ref, message):
|
||||
self.ref, self.message, self.sent = bridge_connection_ref, message, []
|
||||
|
||||
async def send_json(self, request):
|
||||
self.sent.append(request)
|
||||
future = self.ref[0].pending[request["request_id"]]
|
||||
future.set_result({"type": "error", "request_id": request["request_id"], "message": self.message})
|
||||
|
||||
|
||||
def test_load_shedding_does_not_trip_the_breaker():
|
||||
async def scenario():
|
||||
bridge = AIBridge(timeout=1, rng=random.Random(0), breaker_failures=2)
|
||||
ref = []
|
||||
ws = OverloadedAIWebSocket(ref, "overloaded")
|
||||
connection = AIConnection(ws, "fake")
|
||||
ref.append(connection)
|
||||
bridge._connections.append(connection)
|
||||
obs = observation()
|
||||
for _ in range(5):
|
||||
assert await bridge.decide(obs) in obs.legal_actions
|
||||
assert len(ws.sent) == 5 # still asked every time
|
||||
assert bridge.stats()["paused"] is False
|
||||
|
||||
ws.message = "model crashed" # a real failure counts
|
||||
for _ in range(3):
|
||||
await bridge.decide(obs)
|
||||
assert len(ws.sent) == 7 and bridge.stats()["paused"] is True
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_ai_inflight_limit():
|
||||
async def scenario():
|
||||
bridge = AIBridge(timeout=1, rng=random.Random(0), max_inflight=1)
|
||||
@@ -208,7 +242,9 @@ def test_ai_inflight_limit():
|
||||
connection.pending["busy"] = asyncio.get_running_loop().create_future()
|
||||
bridge._connections.append(connection)
|
||||
obs = observation()
|
||||
assert await bridge.decide(obs) in obs.legal_actions
|
||||
for _ in range(10):
|
||||
assert await bridge.decide(obs) in obs.legal_actions
|
||||
assert ws.sent == [] # not even sent: too many requests already waiting
|
||||
assert bridge.stats()["paused"] is False # load shedding is not a failure
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
Reference in New Issue
Block a user