Add resource limits, abuse protection and overload handling

Running games always continue; new work is refused when there is no
capacity; abuse is rejected immediately.

- Docker: CPU/memory/pids caps per container (overridable in .env);
  uvicorn message size, queue and concurrency limits
- Server abuse limits: connections total/per IP (close 1013), message
  rate per connection (token bucket, close 1008 on floods), lobby
  creation rate per IP
- Load monitor: event loop lag, CPU of the own cgroup quota and
  throttling, with hysteresis (normal/busy/overloaded, severe lag
  overloads immediately)
- Admission control: create_lobby and start_game refused while
  overloaded or MAX_ACTIVE_GAMES are running; server_status message
  broadcast on changes; /health reports load but stays ok
- AI bridge: timeout sent with requests, in-flight cap, circuit breaker
- AI service: bounded workers and queue ("overloaded"), drops expired
  requests, degrades to the heuristic agent when busy or too slow
- Website: busy notices, Create/Start disabled when not accepting
  games, slow retry when the server is full
- Tests for limiters, load levels, admission control and AI service
  limits; docs updated

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-09-27 12:51:46 +02:00
co-authored by Claude Opus 5.5
parent 905574e8f1
commit 87dfcd55b7
33 changed files with 1591 additions and 128 deletions
+79
View File
@@ -78,3 +78,82 @@ def test_make_agent_falls_back_without_checkpoint(tmp_path):
def test_heuristic_beats_random():
result = evaluate([HeuristicAgent(), RandomAgent(1), RandomAgent(2), RandomAgent(3)], games=40, seed=0)
assert result.mean_scores[0] > max(result.mean_scores[1:])
# --- Service limits -------------------------------------------------------------
import asyncio # noqa: E402
import threading # noqa: E402
from wizard_ai.service import AIService # noqa: E402
class BlockingAgent(RandomAgent):
"""Blocks in act() until released, to simulate a slow model."""
name = "slow-model"
def __init__(self):
super().__init__(seed=0)
self.release = threading.Event()
def act(self, observation):
self.release.wait(5)
return super().act(observation)
def decide_message(request_id, timeout=5.0):
game = WizardGame(3, seed=1)
return {"type": "decide", "request_id": request_id, "timeout": timeout,
"observation": game.observation(game.current_player).to_dict()}
def test_service_answers_with_agent_name():
service = AIService("ws://unused", HeuristicAgent())
reply = asyncio.run(service.handle_request(decide_message("r1")))
assert reply["type"] == "action" and reply["request_id"] == "r1" and reply["agent"] == "heuristic"
def test_service_refuses_when_queue_is_full_and_degrades_when_busy():
async def scenario():
slow = BlockingAgent()
service = AIService("ws://unused", slow, fallback=HeuristicAgent(), workers=1, max_queue=3, degrade_queue=1)
first = asyncio.create_task(service.handle_request(decide_message("a")))
await asyncio.sleep(0.05) # "a" is computing with the slow agent
second = asyncio.create_task(service.handle_request(decide_message("b")))
third = asyncio.create_task(service.handle_request(decide_message("c")))
await asyncio.sleep(0.05)
refused = await service.handle_request(decide_message("d"))
assert refused == {"type": "error", "request_id": "d", "message": "overloaded"}
slow.release.set()
replies = await asyncio.gather(first, second, third)
# "b" starts while "c" is still waiting -> degraded; "c" runs when nothing waits anymore.
assert [r["agent"] for r in replies] == ["slow-model", "heuristic", "slow-model"]
assert service.in_progress == 0
asyncio.run(scenario())
def test_service_drops_expired_requests():
async def scenario():
slow = BlockingAgent()
service = AIService("ws://unused", slow, workers=1)
first = asyncio.create_task(service.handle_request(decide_message("a")))
await asyncio.sleep(0.05)
waiting = asyncio.create_task(service.handle_request(decide_message("b", timeout=0.1)))
await asyncio.sleep(0.2) # "b" expires while "a" blocks the only worker
slow.release.set()
await first
assert (await waiting)["message"] == "expired"
asyncio.run(scenario())
def test_service_uses_fallback_when_primary_is_too_slow():
async def scenario():
service = AIService("ws://unused", RandomAgent(1), fallback=HeuristicAgent())
service._primary_latency = 3.0 # model needs 3 s, server waits 5 s -> more than half
reply = await service.handle_request(decide_message("a", timeout=5.0))
assert reply["agent"] == "heuristic"
asyncio.run(scenario())