Document limits, load control and performance

- New docs/09-limits-and-performance.md: principles, all protection layers, what players see, configuration reference, measurements, tuning guide, monitoring and how to test - Server chapter summarises the limits and links to chapter 9; circuit breaker description and AI_MAX_INFLIGHT default corrected - Architecture: new files in the layout, design decisions for load handling; docs index lists chapter 9 - Protocol, AI and operations chapters link to chapter 9; test counts - Root and component READMEs describe the limits and settings Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Don't trip the AI circuit breaker on load shedding
2026-09-27 13:04:21 +02:00 · 2026-09-27 13:04:21 +02:00 · 2026-09-27 12:51:46 +02:00
40 changed files with 2009 additions and 144 deletions
@@ -9,3 +9,25 @@ ALLOWED_ORIGINS=
 AI_AGENT=model
 LOG_LEVEL=INFO
 # --- Resource limits (hard caps per container) -------------------------------
 SERVER_CPUS=1.0
 SERVER_MEMORY=512M
 AI_CPUS=1.0
 AI_MEMORY=1G
 WEBSITE_CPUS=0.25
 WEBSITE_MEMORY=64M
 # --- Capacity -----------------------------------------------------------------
 # Games running at the same time; more game starts are refused ("server busy").
 MAX_ACTIVE_GAMES=50
 # Open websocket connections in total / per client IP.
 MAX_CONNECTIONS=500
 MAX_CONNECTIONS_PER_IP=10
 # --- AI service ---------------------------------------------------------------
 # Decisions computed in parallel, waiting requests before refusing, and queue
 # depth from which the cheap heuristic agent is used instead of the model.
 AI_WORKERS=1
 AI_MAX_QUEUE=64
 AI_DEGRADE_QUEUE=8
@@ -33,10 +33,25 @@ The `Observation` / action JSON format is defined once in `wizard_game` and shar
 frontend, the server and the AI, so AI models see exactly the same data in training and in
 production.
 ## Limits and load control
 Wizard shares its host with other services, so its resource use is bounded in layers
 (details: [`docs/09-limits-and-performance.md`](docs/09-limits-and-performance.md)):
 - **Docker caps** per container (server 1 CPU / 512 MB, AI 1 CPU / 1 GB, website 0.25 CPU / 64 MB),
  set in `.env`.
 - **Abuse limits**: connections per IP and in total, messages per second per connection,
  lobbies per IP per minute.
 - **Overload handling**: the server watches event loop lag, CPU and cgroup throttling. While it
  is overloaded, or `MAX_ACTIVE_GAMES` games are running, new lobbies and game starts are refused
  ("server busy" in the UI). Running games always continue.
 - **AI cost bounds**: bounded queue and workers in the AI service, which falls back to the cheap
  heuristic player when busy; circuit breaker and in-flight cap in the server.
 ## Deployment
 ```bash
-cp .env.example .env                                # set AI_SHARED_SECRET
+cp .env.example .env                                # set AI_SHARED_SECRET (limits optional)
 docker network create wizard_website                # if they don't exist yet
 docker network create wizard_api
 docker compose up -d --build
@@ -4,6 +4,12 @@ services:
  wizard_website:
    build: ./wizard_website
    restart: unless-stopped
    deploy:
      resources:
        limits:
          cpus: "${WEBSITE_CPUS:-0.25}"
          memory: ${WEBSITE_MEMORY:-64M}
    pids_limit: 64
    networks:
      - wizard_website
@@ -14,6 +20,16 @@ services:
      AI_SHARED_SECRET: ${AI_SHARED_SECRET:?set AI_SHARED_SECRET in .env}
      ALLOWED_ORIGINS: ${ALLOWED_ORIGINS:-}
      LOG_LEVEL: ${LOG_LEVEL:-INFO}
      MAX_ACTIVE_GAMES: ${MAX_ACTIVE_GAMES:-50}
      MAX_CONNECTIONS: ${MAX_CONNECTIONS:-500}
      MAX_CONNECTIONS_PER_IP: ${MAX_CONNECTIONS_PER_IP:-10}
    # Hard caps so Wizard can never starve other services on the host.
    deploy:
      resources:
        limits:
          cpus: "${SERVER_CPUS:-1.0}"
          memory: ${SERVER_MEMORY:-512M}
    pids_limit: 256
    networks:
      - wizard_api
      - wizard_internal
@@ -31,6 +47,18 @@ services:
      AGENT: ${AI_AGENT:-model}
      MODEL_PATH: /srv/models/latest
      LOG_LEVEL: ${LOG_LEVEL:-INFO}
      AI_WORKERS: ${AI_WORKERS:-1}
      AI_MAX_QUEUE: ${AI_MAX_QUEUE:-64}
      AI_DEGRADE_QUEUE: ${AI_DEGRADE_QUEUE:-8}
      # Numeric libraries (numpy/torch) must not start one thread per host core.
      OMP_NUM_THREADS: ${AI_WORKERS:-1}
      MKL_NUM_THREADS: ${AI_WORKERS:-1}
    deploy:
      resources:
        limits:
          cpus: "${AI_CPUS:-1.0}"
          memory: ${AI_MEMORY:-1G}
    pids_limit: 256
    volumes:
      # Trained checkpoints; replace without rebuilding the image.
      - ./wizard_ai/models:/srv/models:ro
@@ -5,7 +5,7 @@
 | Component | Folder | Language / stack | Runs as | Responsibility |
 |-----------|--------|------------------|---------|----------------|
 | Website | `wizard_website/` | React 19, TypeScript, Vite, CSS modules | static files served by nginx (container port 8082) | UI: start page, lobby, game table |
-| Game server | `wizard_server/app/` | Python 3.13, FastAPI, uvicorn | one process (container port 8083) | sessions, lobbies, running games, asking the AI for moves |
+| Game server | `wizard_server/app/` | Python 3.13, FastAPI, uvicorn | one process (container port 8083) | sessions, lobbies, running games, asking the AI for moves, limits and load control |
 | Game engine | `wizard_server/game_logic/` (package `wizard_game`) | pure Python, no dependencies | library, installed into server and AI images | the rules of Wizard |
 | AI service | `wizard_ai/` | Python, `websockets` | one process, no listening port | chooses moves for AI seats; training tools |
@@ -89,8 +89,8 @@ Important properties of this flow:
 ```
 .
-├── compose.yaml                 # all three containers + networks
+├── compose.yaml                 # all three containers, networks, CPU/memory caps
-├── .env.example                 # template for .env (AI_SHARED_SECRET, …)
+├── .env.example                 # template for .env (secret, resource limits, capacity)
 ├── README.md                    # short project readme
 ├── docs/                        # this documentation
 ├── instructions.txt             # original task description
@@ -107,8 +107,11 @@ Important properties of this flow:
 │   │   ├── service.py           # WizardService: dispatches messages, sessions, cleanup
 │   │   ├── sessions.py          # Session, Connection (ordered send queue), SessionRegistry
 │   │   ├── lobby.py             # Lobby, Seat, LobbyManager, AI turn loop
-│   │   └── ai_bridge.py         # request/response with the AI service + fallback
+│   │   ├── ai_bridge.py         # request/response with the AI service, fallback, circuit breaker
-│   ├── tests/test_server.py     # websocket-level tests of the server
+│   │   ├── limits.py            # TokenBucket, rate and connection limiters
 │   │   └── load.py              # LoadMonitor: detects overload (lag, CPU, cgroup throttling)
 │   ├── tests/                   # conftest.py/helpers.py (fixtures), test_server.py,
 │   │                            # test_protection.py (limits/overload), test_limits.py (units)
 │   └── game_logic/              # installable package "wizard-game"
 │       ├── pyproject.toml
 │       ├── wizard_game/
@@ -130,7 +133,7 @@ Important properties of this flow:
 │   │   ├── agents/              # Agent interface, random, heuristic, model agent
 │   │   ├── model/               # encoding.py (features/actions), policy.py (model interface)
 │   │   ├── training/            # env.py, self_play.py, evaluate.py
-│   │   └── service/client.py    # production websocket client
+│   │   └── service/client.py    # production websocket client (workers, queue, degradation)
 │   └── tests/test_ai.py
 │
 └── wizard_website/
@@ -158,3 +161,6 @@ Important properties of this flow:
 | Session token in `sessionStorage` | Survives page reloads (reconnect to the same seat) but is per tab, so you can test several players in several tabs. |
 | Per-connection send queue | Websocket messages are always sent in order and never concurrently (see `sessions.Connection`). |
 | Relative asset paths + relative API URL in the frontend | The app works under any path prefix (`/wizard/website/` in production, `/` in development) without rebuilding. |
 | Refuse new games under load, never interrupt running ones | Players in a game are never punished for load; creating/starting games is what adds load, so that is what gets refused ("server busy"). Refusing sheds load by itself. See [chapter 9](09-limits-and-performance.md). |
 | Hard Docker caps plus software limits | The caps guarantee the host is protected even if a software limit fails; the software limits make overload visible and graceful instead of just slow. |
 | Health check stays "ok" under load | Docker must never restart an overloaded server — that would end every running game. |
@@ -26,7 +26,9 @@ flowchart TB
 | `service.py` | `WizardService` | the "controller": handles a browser connection, dispatches messages, manages sessions and cleanup |
 | `sessions.py` | `Session`, `Connection`, `SessionRegistry` | player identity across reconnects; ordered sending |
 | `lobby.py` | `LobbyManager`, `Lobby`, `Seat`, `LobbyError` | lobby membership, host actions, running the game, AI turns, broadcasting |
-| `ai_bridge.py` | `AIBridge`, `AIConnection` | talks to the AI service; random fallback |
+| `ai_bridge.py` | `AIBridge`, `AIConnection` | talks to the AI service; random fallback, in-flight cap, circuit breaker |
 | `limits.py` | `TokenBucket`, `KeyedRateLimiter`, `ConnectionLimiter` | abuse limits (message rate, connections, lobby creation) |
 | `load.py` | `LoadMonitor`, `next_level`, cgroup helpers | detects when the server runs out of compute |
 ## Routes (`main.py`)
@@ -249,7 +251,40 @@ The AI service keeps a websocket open to `/internal/ai`. `AIBridge`:
  the action is malformed or illegal (logged as warning, except the
  "no service connected" case which is logged at debug level);
 - calls `on_change` when a service connects/disconnects, and the service then
-  re-broadcasts all lobbies so the lobby screen shows whether the AI is online.
+  re-broadcasts all lobbies so the lobby screen shows whether the AI is online;
 - sends `timeout` with every request so the AI service can drop requests the
  server has already given up on;
 - **in-flight cap**: if `AI_MAX_INFLIGHT` requests are already waiting on a
  connection, the move is played by the fallback without asking;
 - **circuit breaker**: after `AI_BREAKER_FAILURES` real failures in a row
  (timeouts, lost connection, malformed/illegal answers, unexpected errors) the
  AI is not asked for `AI_BREAKER_COOLDOWN` seconds, so games don't wait
  `AI_DECISION_TIMEOUT` for every move of a hanging AI. Load shedding (the
  in-flight cap, AI answers `overloaded` / `expired`) does not count.
 ## Limits and overload handling
 The complete description — all layers, thresholds, what players see, tuning,
 monitoring and measurements — is in
 [chapter 9: Limits, load control and performance](09-limits-and-performance.md).
 In short, for the server:
 - **Rule**: running games always continue; new lobbies/games are refused when
  there is no capacity; abuse is rejected immediately.
 - **Abuse limits** (`limits.py`, applied in `WizardService.handle_client`):
  connections in total and per IP (refused with `error` code `busy` + close
  1013), a token bucket per connection for messages (`rate_limited`, close
  1008 on floods), lobby creation per IP.
 - **Load monitor** (`load.py`, `LoadMonitor.run()` in the app lifespan):
  samples event loop lag, CPU against the container quota, and cgroup
  throttling every second; levels `normal` / `busy` / `overloaded` with
  hysteresis.
 - **Admission control** (`WizardService.accepting_games`): `create_lobby` and
  `start_game` are refused while overloaded or while `MAX_ACTIVE_GAMES` games
  run; everything else keeps working. Changes are broadcast to all clients as
  `server_status`.
 - `/health` reports the load but its `status` stays "ok", so Docker never
  restarts an overloaded server (which would end all games).
 ## Configuration (`config.py`)
@@ -265,27 +300,51 @@ All values are read from environment variables when the module is imported.
 | `MAX_LOBBIES` | 1000 | limit of simultaneously open lobbies |
 | `ALLOWED_ORIGINS` | empty (= all) | comma separated allowed `Origin` headers for `/ws` |
 | `LOG_LEVEL` | INFO | Python logging level |
 | `MAX_CONNECTIONS` / `MAX_CONNECTIONS_PER_IP` | 500 / 10 | open websocket connections |
 | `MESSAGE_RATE` / `MESSAGE_BURST` | 10 / 30 | messages per second per connection |
 | `MAX_RATE_VIOLATIONS` | 50 | dropped messages before a flooding connection is closed |
 | `LOBBY_CREATE_RATE` | 10 | lobbies per IP per minute |
 | `MAX_ACTIVE_GAMES` | 50 | games running at the same time |
 | `LOAD_SAMPLE_INTERVAL` | 1.0 s | load sampling period |
 | `LOAD_LAG_BUSY` / `LOAD_LAG_OVERLOAD` / `LOAD_LAG_SEVERE` | 0.1 / 0.25 / 1.0 s | event loop lag thresholds |
 | `LOAD_CPU_BUSY` / `LOAD_CPU_OVERLOAD` | 0.7 / 0.9 | CPU share of the quota |
 | `LOAD_THROTTLE_OVERLOAD` | 0.5 | throttled share of CPU periods |
 | `LOAD_OVERLOAD_SAMPLES` / `LOAD_RECOVER_SAMPLES` | 3 / 10 | hysteresis |
 | `LOAD_FORCE_LEVEL` | empty | pin the level (testing) |
 | `AI_MAX_INFLIGHT` | 64 | waiting AI requests per AI connection (keep ≥ `MAX_ACTIVE_GAMES`) |
 | `AI_BREAKER_FAILURES` / `AI_BREAKER_COOLDOWN` | 5 / 30 s | AI circuit breaker |
 ## Housekeeping
 `WizardService.cleanup_loop()` runs every 30 s (started in the app lifespan)
-and removes expired sessions (leaving their lobby first).
+and removes expired sessions (leaving their lobby first) and forgets idle
 per-IP lobby-creation buckets.
-## Tests (`wizard_server/tests/test_server.py`)
+## Tests (`wizard_server/tests/`)
-Uses FastAPI's `TestClient` websockets with zero delays. Covers: health,
+Uses FastAPI's `TestClient` websockets with zero delays and a raised message
-hello requirement, validation errors, the full lobby flow including a complete
+rate limit (`helpers.TEST_SETTINGS`).
 20-round game between two humans and an AI, reconnect by token, leaving
 mid-game (seat becomes AI, host moves), the AI endpoint's secret and proxy
 checks, and a fake AI service answering decide requests.
-Note for writing new tests: enter websocket sessions with
+- `test_server.py` — health, hello requirement, validation errors, the full
-`client.sockets.enter_context(client.websocket_connect(...))` (see the fixture),
+  lobby flow including a complete 20-round game between two humans and an AI,
-otherwise the test client hangs on shutdown.
+  reconnect by token, leaving mid-game (seat becomes AI, host moves), the AI
  endpoint's secret and proxy checks, and a fake AI service.
 - `test_protection.py` — message flood, connections per IP, lobby creation
  rate, forced overload, `MAX_ACTIVE_GAMES`, a running game continuing while
  overloaded, `server_status` messages.
 - `test_limits.py` — unit tests of the token bucket, limiters, load levels,
  cgroup parsing and the AI circuit breaker / in-flight cap.
 Fixtures (`conftest.py`): `client` (default test settings) and
 `make_client(**overrides)` for custom settings. Enter websocket sessions with
 `client.sockets.enter_context(client.websocket_connect(...))` (as
 `helpers.connect` does), otherwise the test client hangs on shutdown.
 ## Limitations to be aware of
 - In-memory state: restarting the server ends all lobbies and games.
 - Must run as a single process (no `--workers`).
 - No spectators: only seated players receive game state.
- No rate limiting on websocket messages.
+- A running game whose humans all disconnected keeps its `MAX_ACTIVE_GAMES`
  slot until their sessions expire (`SESSION_TTL`), because they may still
  reconnect. It costs no CPU while waiting.
@@ -93,25 +93,51 @@ contains the **complete** current state.
 - `ai_available` is `false` when no AI service is connected; AI seats then play
  random moves.
 #### `server_status`
 Sent right after `welcome` and to every connected client whenever the server's
 load level or capacity changes.
 ```json
 {"type": "server_status", "load": "overloaded", "accepting_games": false}
 ```
 - `load`: `normal`, `busy` (informational) or `overloaded`.
 - `accepting_games`: `false` while the server refuses `create_lobby` and
  `start_game` (overloaded, or `MAX_ACTIVE_GAMES` games running). Everything
  else, including running games, keeps working.
 #### `error`
 ```json
 {"type": "error", "message": "Only the host can do that"}
 {"type": "error", "message": "The server is busy right now, …", "code": "busy"}
 ```
 `code` is present for errors the client may react to:
 | `code` | Meaning |
 |--------|---------|
 | `busy` | no capacity: connection limit reached (followed by close 1013), or `create_lobby` / `start_game` refused because the server does not accept new games |
 | `rate_limited` | too many messages on this connection, or too many lobbies created from this IP |
 Sent only to the client that caused it. Typical messages: `Send 'hello' first`,
 `Invalid message: …` (followed by the pydantic reason),
 `Lobby ABC123 does not exist`, `The game in this lobby has already started`,
 `This lobby is full`, `Only the host can do that`,
 `Wizard needs 3 to 6 players`, `Illegal action Bid(value=2) for player 0`,
 `It is not player 1's turn`, `You were removed from the lobby`,
-`This session was opened in another window`.
+`This session was opened in another window`, `Too many messages, slow down.`,
 `You are creating lobbies too quickly, please wait a moment.`
 ### Close codes
 | Code | Meaning | Frontend reaction |
 |------|---------|-------------------|
 | (handshake rejected) | origin not in `ALLOWED_ORIGINS`; the server closes with 1008 before accepting, so the browser sees a failed connection | reconnects with backoff |
 | 1008 | message flood (too many dropped messages) | reconnects with backoff |
 | 1009 | message larger than 64 KB (uvicorn `--ws-max-size`) | reconnects with backoff |
 | 1013 | "Try Again Later": connection limit reached (preceded by an `error` with code `busy`) | banner "The server is full", retries slowly (15 s, 30 s, 60 s …) |
 | 4000 | the session was opened in another tab | stops reconnecting, shows a banner |
 | other | network problem, server restart | reconnects (1 s, 2 s, 4 s … max 10 s) |
@@ -153,9 +179,9 @@ headers (see "Routes" in [the server chapter](04-server.md)).
 | Direction | Message | Meaning |
 |-----------|---------|---------|
 | AI → server | `{"type":"hello","agent":"heuristic"}` | informational, logged |
-| server → AI | `{"type":"decide","request_id":"<hex>","observation":{…}}` | choose an action for `observation.player` |
+| server → AI | `{"type":"decide","request_id":"<hex>","timeout":5.0,"observation":{…}}` | choose an action for `observation.player`; the server waits at most `timeout` seconds |
-| AI → server | `{"type":"action","request_id":"<same>","action":{"type":"play","card":17}}` | the chosen action |
+| AI → server | `{"type":"action","request_id":"<same>","action":{"type":"play","card":17},"agent":"heuristic"}` | the chosen action; `agent` names the agent that decided (logged) |
-| AI → server | `{"type":"error","request_id":"<same>","message":"…"}` | the AI failed; the server plays a random move |
+| AI → server | `{"type":"error","request_id":"<same>","message":"…"}` | the AI failed or refused (`overloaded`, `expired`); the server plays a random move |
 Rules:
@@ -163,6 +189,12 @@ Rules:
  (matched by `request_id`) and may handle several concurrently.
 - The answer must be one of `observation.legal_actions`; otherwise the server
  plays a random legal move.
- No answer within `AI_DECISION_TIMEOUT` (5 s) → random move.
+- No answer within `AI_DECISION_TIMEOUT` (5 s) → random move. The AI service
  drops requests whose `timeout` passed before it could start them (`expired`).
 - The server stops asking an AI connection that already has `AI_MAX_INFLIGHT`
  unanswered requests, and pauses the AI after repeated real failures (circuit
  breaker). `overloaded` / `expired` answers are load shedding, not failures,
  and do not trip the breaker. See
  "Layer 4" in [chapter 9](09-limits-and-performance.md).
 - Several AI service instances may connect; the server distributes requests
  round robin.
@@ -53,7 +53,8 @@ The only stateful piece of the app.
 | Value | Meaning |
 |-------|---------|
-| `status` | `connecting` \| `open` \| `closed` (will retry) \| `replaced` (opened in another tab, no retry) |
+| `status` | `connecting` \| `open` \| `closed` (will retry) \| `replaced` (opened in another tab, no retry) \| `busy` (server full, retries slowly) |
 | `serverStatus` | last `server_status` message: `{ load, accepting_games }` |
 | `playerId` | from `welcome` |
 | `lobby`, `game` | exactly what the last `state` message contained |
 | `completedTrick` | the last `trick_completed` event, kept for 2.2 s so the trick stays visible |
@@ -73,6 +74,10 @@ The only stateful piece of the app.
  second tab is a different player.
 - On close it reconnects with exponential backoff (1, 2, 4, 8, 10, 10 … s),
  except after close code 4000.
 - If the server refuses the connection because it is full (an `error` with
  code `busy` before `welcome`, then close code 1013), the status becomes
  `busy` and it retries slowly (15 s, 30 s, 60 s, …) so waiting clients don't
  add load.
 - `send()` shows a toast if the socket is not open.
 ## Screen selection (`App.tsx`)
@@ -96,6 +101,8 @@ shows the connection status when it is not `open`.
 - Code input + "Join" → `actions.joinLobby(code, name)`. The code is prefilled
  from `?lobby=` so invite links work.
 - Buttons are disabled while disconnected or without a name.
 - While `serverStatus.accepting_games` is false, "Create lobby" is disabled and
  a notice explains that the server is busy (joining still works).
 ### LobbyScreen
@@ -107,6 +114,8 @@ shows the connection status when it is not `open`.
  seats are filled). Others see the options read-only and "Waiting for the
  host…".
 - Shows a warning if AI players are present but the AI service is offline.
 - While `serverStatus.accepting_games` is false, "Start game" shows
  "Server busy" (disabled) with a notice; the lobby stays open.
 - Explains the fixed bidding rule (bids may not add up to the cards in hand).
 ### GameScreen (`screens/GameScreen/`)
@@ -194,8 +194,11 @@ registered with `@register_policy`, and calls its `load`.
   the container (`compose.yaml`); restart `wizard_ai`. With `AGENT=model`
   (default) it is loaded automatically.
-Model size matters: the service answers every AI move in a worker thread and
+Model size matters: the service answers every AI move in a worker thread, the
-the server waits at most `AI_DECISION_TIMEOUT` (5 s).
+container is capped at `AI_CPUS` (1 CPU by default) and the server waits at
 most `AI_DECISION_TIMEOUT` (5 s). If the model is too slow the service
 automatically answers with the heuristic instead (see below), so measure the
 decision time before deploying.
 ## Training (`training/`)
@@ -244,14 +247,30 @@ count as a win for everyone tied). Seeded for fair comparisons.
 ## Production service (`service/client.py`)
-`AIService(server_url, agent, secret).run_forever()`:
+`AIService(server_url, agent, secret, fallback=…, workers=…, max_queue=…, degrade_queue=…).run_forever()`:
 - connects to `SERVER_URL` with `Authorization: Bearer <AI_SHARED_SECRET>`,
  sends `hello`, then answers `decide` requests (protocol in
  [chapter 5](05-protocol.md#ai-protocol));
- each request runs `agent.act` in a worker thread (`asyncio.to_thread`) behind
+- each request runs `agent.act` in a worker thread (`asyncio.to_thread`), so a
-  a lock, so a slow model does not block the websocket and non-thread-safe
+  slow model does not block the websocket;
-  agents stay consistent;
+- **bounded work** (`handle_request`):
  - at most `AI_WORKERS` (default 1) decisions are computed at the same time —
    agents must be thread safe if you raise it;
  - at most `AI_MAX_QUEUE` (64) requests may be in progress; more are answered
    with `error` "overloaded" immediately;
  - a request whose server `timeout` passed while it waited for a worker is
    answered with `error` "expired" without computing it;
 - **degradation**: when `AI_DEGRADE_QUEUE` (8) or more other requests are
  waiting, or the primary agent's average decision time is more than half of
  the remaining time, the cheap fallback agent (`HeuristicAgent`) decides
  instead. `serve` sets the heuristic as fallback automatically unless the
  primary agent already is the heuristic. The reply's `agent` field says which
  one decided;
 - `OMP_NUM_THREADS` / `MKL_NUM_THREADS` are set to `AI_WORKERS` in
  `compose.yaml`, so numeric libraries don't start a thread per host core
  (the whole picture, including the server side, is in
  [chapter 9](09-limits-and-performance.md));
 - errors while deciding are reported back as `error` (the server then plays a
  random move) instead of crashing;
 - reconnects forever with exponential backoff (1 s … 30 s); logs a hint if the
@@ -271,10 +290,13 @@ Environment variables (`config.py`): `SERVER_URL`
 (default `ws://localhost:8000/internal/ai`; in Docker
 `ws://wizard_server:8083/internal/ai`), `AI_SHARED_SECRET`, `AGENT`
 (default `model`), `MODEL_PATH` (default `models/latest`, in Docker
-`/srv/models/latest`), `LOG_LEVEL`.
+`/srv/models/latest`), `LOG_LEVEL`, `AI_WORKERS` (1), `AI_MAX_QUEUE` (64),
 `AI_DEGRADE_QUEUE` (8).
 ## Tests (`wizard_ai/tests/test_ai.py`)
 Feature length and masks for 3–6 players, action-space round trip, all agents
 play legal full games, env rewards add up to scores, trainer runs and writes a
-loadable checkpoint, `make_agent` fallback, heuristic beats random.
+loadable checkpoint, `make_agent` fallback, heuristic beats random, and the
 service limits (queue full → overloaded, expired requests, degradation to the
 heuristic when busy or when the model is too slow).
@@ -44,8 +44,16 @@ not read by anything.
 | `ALLOWED_ORIGINS` | server | no | e.g. `https://game.marvinkrausser.com` |
 | `AI_AGENT` | AI | no (default `model`) | `model`, `heuristic` or `random` |
 | `LOG_LEVEL` | server + AI | no | `INFO`, `DEBUG`, … |
 | `SERVER_CPUS` / `SERVER_MEMORY` | compose | no (1.0 / 512M) | hard caps of the server container |
 | `AI_CPUS` / `AI_MEMORY` | compose | no (1.0 / 1G) | hard caps of the AI container |
 | `WEBSITE_CPUS` / `WEBSITE_MEMORY` | compose | no (0.25 / 64M) | hard caps of the website container |
 | `MAX_ACTIVE_GAMES` | server | no (50) | games running at the same time; more starts are refused ("server busy") |
 | `MAX_CONNECTIONS` / `MAX_CONNECTIONS_PER_IP` | server | no (500 / 10) | open websocket connections |
 | `AI_WORKERS` / `AI_MAX_QUEUE` / `AI_DEGRADE_QUEUE` | AI | no (1 / 64 / 8) | parallel decisions, waiting requests before refusing, queue depth from which the heuristic is used |
-Further server settings (`AI_MOVE_DELAY`, `TRICK_PAUSE`, …, see
+How the limits work together, the full list of settings, measurements and a
 tuning guide are in [chapter 9](09-limits-and-performance.md). Further server settings
 (`AI_MOVE_DELAY`, `TRICK_PAUSE`, load thresholds, …, see
 "Configuration" in [the server chapter](04-server.md)) can be added to
 the `environment:` section of `wizard_server` in `compose.yaml`.
@@ -147,8 +155,8 @@ cd wizard_website && npm run lint && npm run build
 | Suite | Count | Covers |
 |-------|-------|--------|
 | game engine | 52 | rules, full random games for 3–6 players, invariants, JSON |
-| server | 9 | websocket flows incl. a complete game, reconnect, AI endpoint security |
+| server | 33 | websocket flows incl. a complete game, reconnect, AI endpoint security, abuse limits, overload handling, load levels |
-| AI | 13 | encoding, agents, env, trainer, checkpoints |
+| AI | 17 | encoding, agents, env, trainer, checkpoints, service limits |
 ## Troubleshooting
@@ -161,6 +169,10 @@ cd wizard_website && npm run lint && npm run build
 | AI players only make weird/random moves | Same as above (random fallback), or the AI answers too slowly (`AI_DECISION_TIMEOUT`), see server logs "Playing random move for player …". |
 | Game waits and nothing happens | It is a disconnected human's turn (seat shows "offline"). The game waits until they reconnect or their session expires (`SESSION_TTL`, then an AI takes over). |
 | All lobbies gone | The server restarted — state is in memory only. |
 | Start page / lobby says "The server is busy", Create/Start disabled | The server is overloaded or `MAX_ACTIVE_GAMES` games are running. Check `https://…/wizard/api/health` (`load`, `cpu`, `throttled`, `active_games`) and `docker stats`. Running games are not affected. If it happens often, raise `SERVER_CPUS` or `MAX_ACTIVE_GAMES`. |
 | Banner "The server is full right now" | Connection limit reached (`MAX_CONNECTIONS`, or `MAX_CONNECTIONS_PER_IP` — e.g. many tabs behind one NAT). The page retries on its own every 15–60 s. |
 | Error toast "Too many messages, slow down." | A client exceeded the message rate limit (10/s). Normal play never gets near it; a script or broken client might. |
 | AI plays noticeably worse under load | The AI service switched to the heuristic because it was busy (`AI_DEGRADE_QUEUE`) or the model was too slow; see `docker compose logs wizard_ai` ("Busy … using heuristic agent"). Raise `AI_CPUS`/`AI_WORKERS` or use a smaller model. |
 | Tests pass but behaviour does not change after editing `wizard_game` | A non-editable copy of `wizard_game` is installed in your virtualenv and shadows the source. `pip uninstall wizard-game && pip install -e wizard_server/game_logic`. Check with `python -c "import wizard_game; print(wizard_game.__file__)"`. |
 | Server tests hang | A test opened a websocket without a context manager; use `client.sockets.enter_context(client.websocket_connect(...))`. |
@@ -190,6 +202,17 @@ cd wizard_website && npm run lint && npm run build
   `actions`.
 4. Add a test in `wizard_server/tests/test_server.py`.
 ### Check resource usage
 ```bash
 docker stats wizard-wizard_server-1 wizard-wizard_ai-1 wizard-wizard_website-1
 curl -s https://game.marvinkrausser.com/wizard/api/health
 ```
 To test the overload behaviour without real load, start the server with
 `LOAD_FORCE_LEVEL=overloaded` (new games are refused, the UI shows the busy
 notice).
 ### Change how fast AI players move
 Set `AI_MOVE_DELAY` (per move) and `TRICK_PAUSE` (after a trick) for
@@ -0,0 +1,373 @@
 # 9. Limits, load control and performance
 This chapter explains how Wizard keeps its resource use bounded, how it
 protects itself against abuse, and what happens when it runs out of compute.
 It covers all components, because the protection works in layers from Docker
 down to the browser.
 ## Principles
 1. **Running games always continue.** A game in progress is never stopped,
   slowed down on purpose or refused a move because of load.
 2. **New work is refused when there is no capacity.** Creating lobbies and
   starting games is what adds load, so that is what gets refused — with a
   clear "server busy" message, not a timeout. (A waiting queue was considered
   and rejected as not worth the complexity.)
 3. **Abuse is rejected immediately.** Floods and connection storms are cut off
   at the door, never queued.
 4. **The host is protected no matter what.** Hard Docker caps guarantee that
   Wizard can never take CPU or memory from the other services on the machine,
   even if every software limit fails.
 5. **Degrade, don't fail.** An overloaded or broken AI makes weaker moves (the
   heuristic, or a random move) instead of stopping games.
 ## Overview
 ```mermaid
 flowchart TB
    subgraph docker["Layer 1: Docker caps (compose.yaml)"]
        direction TB
        subgraph server["wizard_server — 1 CPU / 512 MB"]
            U["uvicorn: max message 64 KB,<br/>max 1000 connections"]
            A["Layer 2: abuse limits<br/>connections, message rate,<br/>lobby creation (limits.py)"]
            L["Layer 3: load monitor +<br/>admission control (load.py)"]
            B["AI bridge: in-flight cap,<br/>circuit breaker (ai_bridge.py)"]
        end
        subgraph ai["wizard_ai — 1 CPU / 1 GB"]
            Q["Layer 4: workers, queue,<br/>deadlines, degradation<br/>(service/client.py)"]
        end
        W["wizard_website — 0.25 CPU / 64 MB"]
    end
    Browser -->|websocket| U --> A --> L
    L -->|AI turns| B -->|decide| Q
    L -. "server_status" .-> Browser
 ```
 ## Layer 1 — Docker caps
 Set in `compose.yaml` from `.env` (defaults shown):
 | Container | CPU | Memory | Processes |
 |-----------|-----|--------|-----------|
 | `wizard_server` | `SERVER_CPUS=1.0` | `SERVER_MEMORY=512M` | 256 |
 | `wizard_ai` | `AI_CPUS=1.0` | `AI_MEMORY=1G` | 256 |
 | `wizard_website` | `WEBSITE_CPUS=0.25` | `WEBSITE_MEMORY=64M` | 64 |
 - **CPU** is a hard quota: when the container uses it up, the kernel
  throttles it. The server notices this (see layer 3) and refuses new games.
 - **Memory** is a hard limit: a container exceeding it is killed by the kernel
  and restarted by Docker (`restart: unless-stopped`). For the server this
  would end all games (state is in memory), so the limit has lots of headroom:
  the server uses ~80 MB idle and ~100 MB with 100 games (see
  [measurements](#measurements)).
 - **Processes** (`pids_limit`) stop fork bombs / thread explosions.
 - The AI container sets `OMP_NUM_THREADS` / `MKL_NUM_THREADS` to `AI_WORKERS`,
  so numeric libraries (numpy, torch) don't start one thread per *host* core
  inside a 1-CPU container.
 uvicorn (`wizard_server/Dockerfile`) adds transport limits:
 `--ws-max-size 65536` (larger messages close the connection with code 1009;
 real messages are < 1 KB), `--ws-max-queue 16`, `--limit-concurrency 1000`
 (HTTP 503 beyond that — a last resort above the app's own connection limit).
 ## Layer 2 — abuse limits (server)
 `wizard_server/app/limits.py`, applied in `WizardService.handle_client`
 (`service.py`):
 | Limit | Setting (default) | When exceeded |
 |-------|-------------------|---------------|
 | Open connections in total | `MAX_CONNECTIONS` (500) | `error` code `busy`, close **1013** "Try Again Later" |
 | Open connections per client IP | `MAX_CONNECTIONS_PER_IP` (10) | same |
 | Messages per connection | `MESSAGE_RATE` (10/s), `MESSAGE_BURST` (30) | message dropped; `error` code `rate_limited` (only on the 1st and every 10th drop) |
 | Flooding | `MAX_RATE_VIOLATIONS` (50 drops) | connection closed with **1008** |
 | Lobbies created per client IP | `LOBBY_CREATE_RATE` (10 per minute) | `error` code `rate_limited` |
 Building blocks:
 - `TokenBucket(rate, burst)` — the classic algorithm: a bucket holds up to
  `burst` tokens, refills at `rate` per second, each action takes one token.
  Allows short bursts (a player clicking quickly) but bounds the average.
 - `KeyedRateLimiter` — one bucket per key (client IP); buckets that refilled
  completely are forgotten by the cleanup task every 30 s.
 - `ConnectionLimiter` — counts open connections in total and per IP; released
  in the `finally` block of `handle_client`, so crashes cannot leak slots.
 **Client IP**: `websocket.client.host`. Behind Caddy, uvicorn's
 `--proxy-headers` sets it from `X-Forwarded-For`; Caddy ignores
 client-supplied `X-Forwarded-For` values, so the IP cannot be spoofed. In
 local development everything comes from `127.0.0.1`.
 Normal play needs about one message per second, so these limits are never
 noticeable for humans. Several players behind one NAT (e.g. a household)
 share the per-IP connection limit — raise `MAX_CONNECTIONS_PER_IP` if that is
 a realistic use case.
 ## Layer 3 — load monitor and admission control (server)
 ### Detecting overload (`load.py`)
 `LoadMonitor.run()` is started in the app lifespan and takes a sample every
 `LOAD_SAMPLE_INTERVAL` (1 s):
 | Signal | How it is measured | Why |
 |--------|--------------------|-----|
 | **Event loop lag** | the monitor sleeps 1 s and measures how much later it woke up | the most direct measure of "the server reacts slowly"; works everywhere |
 | **CPU** | process CPU time per second, relative to the container's CPU quota (at most 1 core, because game logic runs on one core) | detects saturation before players notice lag |
 | **Throttling** | `nr_throttled / nr_periods` from `cpu.stat` | share of scheduling periods in which the container hit its Docker CPU cap |
 CPU quota and throttling come from the process's **own cgroup** (cgroup v2):
 `/proc/self/cgroup` gives its path, e.g. `0::/` inside a container (→
 `/sys/fs/cgroup`) or `0::/user.slice/…scope` on a host. Without cgroup files
 (e.g. macOS) the monitor still works with lag and CPU against one core.
 Each sample is classified (`classify`), and `next_level` turns the recent
 classifications into the current level with hysteresis:
 | Level | Entered | Left |
 |-------|---------|------|
 | `normal` | — | — |
 | `busy` | one sample with lag ≥ `LOAD_LAG_BUSY` (0.1 s) or CPU ≥ `LOAD_CPU_BUSY` (70 %) | after `LOAD_RECOVER_SAMPLES` (10) normal samples in a row |
 | `overloaded` | `LOAD_OVERLOAD_SAMPLES` (3) samples in a row with lag ≥ `LOAD_LAG_OVERLOAD` (0.25 s), CPU ≥ `LOAD_CPU_OVERLOAD` (90 %) or throttled ≥ `LOAD_THROTTLE_OVERLOAD` (50 %) — **or one** sample with lag ≥ `LOAD_LAG_SEVERE` (1 s) | after 10 samples in a row that are not overloaded |
 Why hysteresis: without it a server at the edge would flip between states
 every second, and every flip is broadcast to all players. Why the "severe"
 rule: when the event loop is extremely late, samples themselves arrive late
 (one per several seconds), so waiting for three would delay detection a lot.
 `busy` is informational only. `LOAD_FORCE_LEVEL` pins the level for testing.
 ### Admission control (`service.py`)
 `WizardService.accepting_games` is true unless the level is `overloaded` or
 `MAX_ACTIVE_GAMES` (50) games are running (`LobbyManager.active_games()`).
 | Action | While not accepting games |
 |--------|---------------------------|
 | `create_lobby` | refused — `error` code `busy` |
 | `start_game` | refused — `error` code `busy` |
 | `join_lobby` | allowed (cheap) |
 | moves in running games, `return_to_lobby`, leaving | allowed (protects running games) |
 | reconnects (`hello` with token) | allowed |
 Refusing new games sheds load by itself: finished games are not replaced, so
 the load falls and the server recovers without intervention.
 ### Telling the players
 Every connected client gets a `server_status` message right after `welcome`
 and whenever the level or `accepting_games` changes:
 ```json
 {"type": "server_status", "load": "overloaded", "accepting_games": false}
 ```
 Changes come from the load monitor (`on_change`) and from lobbies
 (`on_status_change` when a game starts, finishes or is abandoned), and are
 de-duplicated so an unchanged status is not re-sent.
 ### Health endpoint
 `GET /health` (public: `https://game.marvinkrausser.com/wizard/api/health`):
 ```json
 {
  "status": "ok",
  "load": "normal", "loop_lag": 0.0003, "cpu": 0.012, "throttled": 0.0,
  "accepting_games": true,
  "lobbies": 4, "active_games": 3, "sessions": 9, "connections": 7,
  "ai_available": true,
  "ai": {"connections": 1, "pending": 0, "paused": false}
 }
 ```
 `status` **stays "ok" under load** on purpose: the Docker health check must
 never mark an overloaded server unhealthy, because a restart would end every
 running game.
 ## Layer 4 — AI cost bounds
 ### In the server (`ai_bridge.py`)
 | Mechanism | Setting (default) | Behaviour |
 |-----------|-------------------|-----------|
 | Deadline | `AI_DECISION_TIMEOUT` (5 s) | sent as `timeout` in every request; no answer in time → random move |
 | In-flight cap | `AI_MAX_INFLIGHT` (64) | if that many requests already wait on an AI connection, the move is played randomly without asking. Keep it ≥ `MAX_ACTIVE_GAMES`: every game has at most one AI deciding at a time |
 | Circuit breaker | `AI_BREAKER_FAILURES` (5), `AI_BREAKER_COOLDOWN` (30 s) | after 5 **real failures** in a row (timeouts, lost connection, malformed/illegal answers, unexpected errors) the AI is not asked for 30 s; games play random AI moves instead of waiting 5 s per move |
 What counts as a failure matters: the breaker is for a *broken or hanging* AI
 service. Deliberate load shedding — the in-flight cap, or the AI answering
 `overloaded` / `expired` — is **not** a failure: the service is healthy and
 answered immediately, and pausing it would only make play worse.
 ### In the AI service (`wizard_ai/wizard_ai/service/client.py`)
 ```mermaid
 flowchart TD
    R[decide request] --> Q{"in progress ≥<br/>AI_MAX_QUEUE?"}
    Q -- yes --> E1["error: overloaded"]
    Q -- no --> W[wait for a free worker<br/>AI_WORKERS]
    W --> X{"server timeout<br/>already passed?"}
    X -- yes --> E2["error: expired"]
    X -- no --> D{"≥ AI_DEGRADE_QUEUE others waiting,<br/>or primary agent slower than<br/>half the remaining time?"}
    D -- yes --> H[heuristic agent]
    D -- no --> M[primary agent / model]
    H --> OK["action + agent name"]
    M --> OK
 ```
 | Setting (default) | Meaning |
 |-------------------|---------|
 | `AI_WORKERS` (1) | decisions computed at the same time; agents must be thread safe if > 1 |
 | `AI_MAX_QUEUE` (64) | requests in progress (waiting + computing) before refusing with `overloaded` |
 | `AI_DEGRADE_QUEUE` (8) | from this many other waiting requests, the heuristic decides instead of the model |
 The service also keeps a smoothed average of the primary agent's decision
 time; if it exceeds half of the time left before the server's deadline, the
 heuristic answers instead. The reply's `agent` field says who decided (the
 server logs it at debug level). With the heuristic as primary agent (today's
 default without a trained model) there is nothing cheaper to fall back to, so
 degradation only matters once a real model is deployed.
 ## What players see
 | Situation | Start page | Lobby | Running game |
 |-----------|-----------|-------|--------------|
 | Normal | — | — | — |
 | Not accepting games (overloaded or game limit) | notice "The server is busy…", *Create lobby* disabled, *Join* works | notice, *Start game* shows "Server busy" (disabled) | unchanged |
 | Connection limit reached | red banner "The server is full right now. Retrying in a moment…", retries after 15 s, 30 s, 60 s … | — | — |
 | Message rate exceeded | toast "Too many messages, slow down." | same | same |
 | AI overloaded / broken | — | — | AI players make weaker (heuristic or random) moves |
 Implemented in `useWizard.ts` (`serverStatus`, connection status `busy`),
 `App.tsx`, `HomeScreen.tsx` and `LobbyScreen.tsx`.
 ## Configuration reference
 All settings with their defaults. Server settings are read in
 `wizard_server/app/config.py`, AI settings in `wizard_ai/wizard_ai/config.py`.
 Settings marked *compose* are already wired through `compose.yaml` / `.env`;
 the others can be added to the `environment:` section of the service.
 | Variable | Component | Default | Wired in compose |
 |----------|-----------|---------|------------------|
 | `SERVER_CPUS` / `SERVER_MEMORY` | Docker | 1.0 / 512M | yes |
 | `AI_CPUS` / `AI_MEMORY` | Docker | 1.0 / 1G | yes |
 | `WEBSITE_CPUS` / `WEBSITE_MEMORY` | Docker | 0.25 / 64M | yes |
 | `MAX_CONNECTIONS` / `MAX_CONNECTIONS_PER_IP` | server | 500 / 10 | yes |
 | `MESSAGE_RATE` / `MESSAGE_BURST` | server | 10 / 30 | no |
 | `MAX_RATE_VIOLATIONS` | server | 50 | no |
 | `LOBBY_CREATE_RATE` | server | 10 per minute | no |
 | `MAX_LOBBIES` | server | 1000 | no |
 | `MAX_ACTIVE_GAMES` | server | 50 | yes |
 | `LOAD_SAMPLE_INTERVAL` | server | 1.0 s | no |
 | `LOAD_LAG_BUSY` / `LOAD_LAG_OVERLOAD` / `LOAD_LAG_SEVERE` | server | 0.1 / 0.25 / 1.0 s | no |
 | `LOAD_CPU_BUSY` / `LOAD_CPU_OVERLOAD` | server | 0.7 / 0.9 | no |
 | `LOAD_THROTTLE_OVERLOAD` | server | 0.5 | no |
 | `LOAD_OVERLOAD_SAMPLES` / `LOAD_RECOVER_SAMPLES` | server | 3 / 10 | no |
 | `LOAD_FORCE_LEVEL` | server | empty | no |
 | `AI_DECISION_TIMEOUT` | server | 5 s | no |
 | `AI_MAX_INFLIGHT` | server | 64 | no |
 | `AI_BREAKER_FAILURES` / `AI_BREAKER_COOLDOWN` | server | 5 / 30 s | no |
 | `AI_WORKERS` / `AI_MAX_QUEUE` / `AI_DEGRADE_QUEUE` | AI | 1 / 64 / 8 | yes |
 ## Measurements
 Measured on a development machine (uvicorn, heuristic AI, normal pacing:
 `AI_MOVE_DELAY` 0.8 s, one scripted human per game taking 1.5 s per move,
 4 players per game):
 | Running games | Server CPU | Server memory | AI service CPU | AI memory |
 |---------------|-----------|---------------|----------------|-----------|
 | 0 | ~0 % | ~55–80 MB | ~0 % | ~30 MB |
 | 50 | 0.8 % of a core | ~88 MB | 0.5 % | ~30 MB |
 | 100 | 1.5 % of a core | ~99 MB | 0.9 % | ~30 MB |
 Overload test: server limited to **15 % of one CPU**
 (`systemd-run --user --scope -p CPUQuota=15%`), 30 games playing at full speed
 (no delays, instant human moves):
 | Time | Event |
 |------|-------|
 | 0 s | load starts |
 | ~1 s | `busy` |
 | ~3 s | `overloaded` (CPU at 100 % of quota, throttled 100 %) → `create_lobby` refused with `busy` |
 | … | running games keep playing (534 moves made while overloaded) |
 | ~14 s | games that finished could not restart → load gone → back to `normal` |
 What this means:
 - With realistic pacing a game costs almost nothing; **1 CPU is far more than
  50 games need**. `MAX_ACTIVE_GAMES=50` is a conservative safety cap, not a
  CPU requirement — raising it to a few hundred is fine with the heuristic AI.
 - Overload only happens with abnormal traffic (scripts, floods) or a heavy AI
  model. That is exactly what the limits are for.
 - A neural-network model will be the most expensive part. Measure its
  decision time and CPU with `python -m wizard_ai evaluate` and `docker stats`
  before raising the caps.
 ## Tuning guide
 | Symptom | Adjust |
 |---------|--------|
 | "Server busy" although `/health` shows low `cpu` and `load: normal` | the game limit is reached: raise `MAX_ACTIVE_GAMES` (and `AI_MAX_INFLIGHT` to at least the same value) |
 | "Server busy" with high `cpu` / `throttled` | give the server more CPU (`SERVER_CPUS`) — or accept it: the server is protecting itself |
 | Friends behind one router get "server full" | raise `MAX_CONNECTIONS_PER_IP` |
 | Busy/overloaded flips too eagerly on a slow machine | raise `LOAD_LAG_*` thresholds or `LOAD_OVERLOAD_SAMPLES` |
 | AI plays weaker under load (logs: "Busy … using heuristic agent") | raise `AI_CPUS` and `AI_WORKERS` (thread-safe agent!) or use a smaller model; `AI_DEGRADE_QUEUE` sets how early it degrades |
 | Server logs "AI failed 5 times in a row, pausing AI requests" | the AI service hangs or crashes: check `docker compose logs wizard_ai` |
 | Server logs "… AI requests already waiting" | more simultaneous AI turns than `AI_MAX_INFLIGHT`: raise it to ≥ `MAX_ACTIVE_GAMES` |
 | Container restarts, `docker inspect` shows `OOMKilled: true` | raise `SERVER_MEMORY` / `AI_MEMORY` |
 ## Monitoring
 - `curl -s https://game.marvinkrausser.com/wizard/api/health` — load, CPU,
  games, connections, AI state (see above).
 - `docker stats` — live CPU and memory per container against the caps.
 - Log lines to watch (`docker compose logs -f wizard_server wizard_ai`):
  | Log line | Meaning |
  |----------|---------|
  | `Load level normal -> busy` / `busy -> overloaded` / `-> normal` | load level changes (with lag, CPU, throttling) |
  | `Refusing connection from <ip>: connection limit reached` | connection limits hit |
  | `Closing connection from <ip>: message flood` | a client was cut off |
  | `AI failed N times in a row, pausing AI requests for 30s` | circuit breaker opened |
  | `Playing random move for player N: …` | an AI move fell back to random (reason included) |
  | AI: `Overloaded: N requests in progress` / `Dropping expired request` / `Busy (N waiting): using heuristic agent` | the AI service shedding load |
 ## Testing it yourself
 Automated tests:
 - `wizard_server/tests/test_limits.py` — token bucket, limiters, load levels
  and hysteresis, cgroup parsing, AI circuit breaker and in-flight cap.
 - `wizard_server/tests/test_protection.py` — floods, connection limits, lobby
  creation rate, forced overload, `MAX_ACTIVE_GAMES`, a game continuing while
  overloaded, `server_status` messages.
 - `wizard_ai/tests/test_ai.py` — queue full, expired requests, degradation.
 Manual checks:
 ```bash
 # Refusal UI without real load
 cd wizard_server && LOAD_FORCE_LEVEL=overloaded AI_SHARED_SECRET=dev uvicorn app.main:app
 # Game limit: start one game, then a second lobby's "Start game" is refused
 cd wizard_server && MAX_ACTIVE_GAMES=1 AI_SHARED_SECRET=dev uvicorn app.main:app
 # Real CPU exhaustion on Linux: cap the server at 15 % of a CPU,
 # then run many games without delays (e.g. a small websocket script)
 cd wizard_server && AI_MOVE_DELAY=0 TRICK_PAUSE=0 MAX_CONNECTIONS_PER_IP=200 \
  MESSAGE_RATE=1000 MESSAGE_BURST=1000 AI_SHARED_SECRET=dev \
  systemd-run --user --scope -p CPUQuota=15% uvicorn app.main:app
 ```
 ## Known limitations
 - A running game whose humans all disconnected keeps its `MAX_ACTIVE_GAMES`
  slot until their sessions expire (`SESSION_TTL`, 30 min), because they may
  reconnect. It costs no CPU meanwhile.
 - The limits are per server process. That is fine because the server must run
  as a single process anyway (all state is in memory).
 - Players behind one IP share `MAX_CONNECTIONS_PER_IP` and `LOBBY_CREATE_RATE`.
 - There is no waiting queue: players have to retry when the server is busy.
@@ -11,11 +11,12 @@ to be read top to bottom the first time and used as a reference afterwards.
 | 1 | [Architecture overview](01-architecture.md) | The four components, how a move travels through the system, repository layout, key design decisions |
 | 2 | [Game rules as implemented](02-game-rules.md) | Every rule of Wizard as the code implements it, with the exact function that enforces it |
 | 3 | [Game engine (`wizard_game`)](03-game-engine.md) | The rules package: cards, the `WizardGame` state machine, observations, actions, events |
-| 4 | [Game server (`wizard_server/app`)](04-server.md) | Sessions, reconnects, lobbies, running games, AI turns, broadcasting, configuration |
+| 4 | [Game server (`wizard_server/app`)](04-server.md) | Sessions, reconnects, lobbies, running games, AI turns, broadcasting, limits (summary), configuration |
 | 5 | [Protocol reference](05-protocol.md) | Every websocket message between browser ↔ server and server ↔ AI, with examples |
 | 6 | [Frontend (`wizard_website`)](06-frontend.md) | The React app: connection hook, screens, components, styling, build |
 | 7 | [AI player (`wizard_ai`)](07-ai.md) | Agents, feature encoding, policy interface, training, evaluation, production service — and how to plug in a real model |
 | 8 | [Deployment, development & operations](08-operations.md) | Docker, networks, Caddy, environment variables, local setup, tests, troubleshooting, how-to recipes |
 | 9 | [Limits, load control and performance](09-limits-and-performance.md) | Resource caps, abuse limits, overload detection, what happens when compute runs out, AI cost bounds, measurements, tuning and monitoring |
 ## The system in one paragraph
@@ -16,12 +16,16 @@ provides everything around it.
  `Policy.update`, with checkpointing
 - `training/evaluate.py` — play agents against each other (mean score, win rate)
 - `service/client.py` — production: connects to `ws://wizard_server:8083/internal/ai`,
-  answers `decide` requests, reconnects automatically
+  answers `decide` requests, reconnects automatically. Work is bounded: `AI_WORKERS` (1)
  parallel decisions, at most `AI_MAX_QUEUE` (64) requests in progress (more are answered
  "overloaded"), expired requests are dropped, and from `AI_DEGRADE_QUEUE` (8) waiting requests —
  or when the model is too slow — the heuristic decides instead of the model
  (see `../docs/09-limits-and-performance.md`)
 ```bash
 pip install -r requirements-dev.txt
 python -m wizard_ai train --episodes 5000 --out models/latest
 python -m wizard_ai evaluate --agents model,heuristic,random,random --games 200
-python -m wizard_ai serve   # env: SERVER_URL, AI_SHARED_SECRET, AGENT, MODEL_PATH
+python -m wizard_ai serve   # env: SERVER_URL, AI_SHARED_SECRET, AGENT, MODEL_PATH, AI_WORKERS, AI_MAX_QUEUE, AI_DEGRADE_QUEUE
 pytest
 ```
@@ -78,3 +78,82 @@ def test_make_agent_falls_back_without_checkpoint(tmp_path):
 def test_heuristic_beats_random():
    result = evaluate([HeuristicAgent(), RandomAgent(1), RandomAgent(2), RandomAgent(3)], games=40, seed=0)
    assert result.mean_scores[0] > max(result.mean_scores[1:])
 # --- Service limits -------------------------------------------------------------
 import asyncio  # noqa: E402
 import threading  # noqa: E402
 from wizard_ai.service import AIService  # noqa: E402
 class BlockingAgent(RandomAgent):
    """Blocks in act() until released, to simulate a slow model."""
    name = "slow-model"
    def __init__(self):
        super().__init__(seed=0)
        self.release = threading.Event()
    def act(self, observation):
        self.release.wait(5)
        return super().act(observation)
 def decide_message(request_id, timeout=5.0):
    game = WizardGame(3, seed=1)
    return {"type": "decide", "request_id": request_id, "timeout": timeout,
            "observation": game.observation(game.current_player).to_dict()}
 def test_service_answers_with_agent_name():
    service = AIService("ws://unused", HeuristicAgent())
    reply = asyncio.run(service.handle_request(decide_message("r1")))
    assert reply["type"] == "action" and reply["request_id"] == "r1" and reply["agent"] == "heuristic"
 def test_service_refuses_when_queue_is_full_and_degrades_when_busy():
    async def scenario():
        slow = BlockingAgent()
        service = AIService("ws://unused", slow, fallback=HeuristicAgent(), workers=1, max_queue=3, degrade_queue=1)
        first = asyncio.create_task(service.handle_request(decide_message("a")))
        await asyncio.sleep(0.05)  # "a" is computing with the slow agent
        second = asyncio.create_task(service.handle_request(decide_message("b")))
        third = asyncio.create_task(service.handle_request(decide_message("c")))
        await asyncio.sleep(0.05)
        refused = await service.handle_request(decide_message("d"))
        assert refused == {"type": "error", "request_id": "d", "message": "overloaded"}
        slow.release.set()
        replies = await asyncio.gather(first, second, third)
        # "b" starts while "c" is still waiting -> degraded; "c" runs when nothing waits anymore.
        assert [r["agent"] for r in replies] == ["slow-model", "heuristic", "slow-model"]
        assert service.in_progress == 0
    asyncio.run(scenario())
 def test_service_drops_expired_requests():
    async def scenario():
        slow = BlockingAgent()
        service = AIService("ws://unused", slow, workers=1)
        first = asyncio.create_task(service.handle_request(decide_message("a")))
        await asyncio.sleep(0.05)
        waiting = asyncio.create_task(service.handle_request(decide_message("b", timeout=0.1)))
        await asyncio.sleep(0.2)  # "b" expires while "a" blocks the only worker
        slow.release.set()
        await first
        assert (await waiting)["message"] == "expired"
    asyncio.run(scenario())
 def test_service_uses_fallback_when_primary_is_too_slow():
    async def scenario():
        service = AIService("ws://unused", RandomAgent(1), fallback=HeuristicAgent())
        service._primary_latency = 3.0  # model needs 3 s, server waits 5 s -> more than half
        reply = await service.handle_request(decide_message("a", timeout=5.0))
        assert reply["agent"] == "heuristic"
    asyncio.run(scenario())
@@ -49,10 +49,20 @@ def main() -> None:
    logging.basicConfig(level=args.log_level.upper(), format="%(asctime)s %(levelname)s %(name)s: %(message)s")
    if args.command == "serve":
        from .agents import HeuristicAgent
        from .service import AIService
        agent = make_agent(args.agent, args.model_path)
-        asyncio.run(AIService(args.server_url, agent, config.secret).run_forever())
+        # Cheap agent used when the service is busy (pointless if it already is the heuristic).
        fallback = None if isinstance(agent, HeuristicAgent) else HeuristicAgent()
        service = AIService(
            args.server_url, agent, config.secret,
            fallback=fallback,
            workers=config.workers,
            max_queue=config.max_queue,
            degrade_queue=config.degrade_queue,
        )
        asyncio.run(service.run_forever())
    elif args.command == "train":
        from .model import UniformPolicy, checkpoint_exists, load_policy
@@ -17,3 +17,9 @@ class Config:
    # Checkpoint directory of the trained policy.
    model_path: str = os.environ.get("MODEL_PATH", "models/latest")
    log_level: str = os.environ.get("LOG_LEVEL", "INFO")
    # Decisions computed in parallel (agents must be thread safe if > 1).
    workers: int = int(os.environ.get("AI_WORKERS", 1))
    # Requests allowed to wait; more are refused with "overloaded".
    max_queue: int = int(os.environ.get("AI_MAX_QUEUE", 64))
    # Waiting requests from which the heuristic agent is used instead of the model.
    degrade_queue: int = int(os.environ.get("AI_DEGRADE_QUEUE", 8))
@@ -3,6 +3,18 @@
 The service connects to the server's ``/internal/ai`` endpoint and answers
 ``decide`` requests (see ``wizard_server/app/ai_bridge.py``). It reconnects
 automatically if the connection drops.
 Work is bounded so a busy service degrades instead of falling over:
 - at most ``workers`` decisions are computed at the same time;
 - at most ``max_queue`` requests may wait; more are answered with the error
  "overloaded" right away (the server then plays a fallback move);
 - a request that waited longer than the server's ``timeout`` is answered with
  "expired" without computing it (the server has already moved on);
 - when ``degrade_queue`` or more other requests are waiting, or the primary
  agent typically needs more than half of the remaining time, the cheap
  ``fallback`` agent (heuristic) decides instead of the primary agent (e.g. a
  neural network).
 """
 from __future__ import annotations
@@ -10,6 +22,8 @@ from __future__ import annotations
 import asyncio
 import json
 import logging
 import time
 from collections.abc import Callable
 from typing import Any
 from websockets.asyncio.client import ClientConnection, connect
@@ -22,16 +36,36 @@ from ..agents import Agent
 logger = logging.getLogger(__name__)
 MAX_BACKOFF = 30.0
 # Weight of the newest measurement in the average decision time.
 LATENCY_SMOOTHING = 0.2
 class AIService:
-    def __init__(self, server_url: str, agent: Agent, secret: str = "") -> None:
+    def __init__(
        self,
        server_url: str,
        agent: Agent,
        secret: str = "",
        *,
        fallback: Agent | None = None,
        workers: int = 1,
        max_queue: int = 64,
        degrade_queue: int = 8,
        clock: Callable[[], float] = time.monotonic,
    ) -> None:
        self.server_url = server_url
        self.agent = agent
        self.fallback = fallback
        self.secret = secret
-        # Inference runs in worker threads; the lock keeps agents that are not
+        self.max_queue = max_queue
-        # thread safe (e.g. stateful models) consistent.
+        self.degrade_queue = degrade_queue
-        self._agent_lock = asyncio.Lock()
+        self._clock = clock
        # Agents run in worker threads; with workers > 1 they must be thread safe.
        self._workers = asyncio.Semaphore(max(1, workers))
        # Requests received and not answered yet (waiting + computing).
        self.in_progress = 0
        # Smoothed seconds the primary agent needs per decision.
        self._primary_latency = 0.0
    async def run_forever(self) -> None:
        backoff = 1.0
@@ -55,21 +89,62 @@ class AIService:
            async for raw in websocket:
                message = json.loads(raw)
                if message.get("type") == "decide":
-                    task = asyncio.create_task(self._decide(websocket, message))
+                    task = asyncio.create_task(self._answer(websocket, message))
                    tasks.add(task)
                    task.add_done_callback(tasks.discard)
-    async def _decide(self, websocket: ClientConnection, message: dict[str, Any]) -> None:
+    async def _answer(self, websocket: ClientConnection, message: dict[str, Any]) -> None:
-        request_id = message.get("request_id")
+        reply = await self.handle_request(message)
        try:
            observation = Observation.from_dict(message["observation"])
            async with self._agent_lock:
                action = await asyncio.to_thread(self.agent.act, observation)
            reply = {"type": "action", "request_id": request_id, "action": action.to_dict()}
        except Exception as exc:  # report instead of crashing; the server falls back
            logger.exception("Failed to decide request %s", request_id)
            reply = {"type": "error", "request_id": request_id, "message": str(exc)}
        try:
            await websocket.send(json.dumps(reply))
        except ConnectionClosed:
            pass
    async def handle_request(self, message: dict[str, Any]) -> dict[str, Any]:
        """Answer one ``decide`` request (independent of the websocket, for tests)."""
        request_id = message.get("request_id")
        received = self._clock()
        timeout = message.get("timeout")
        deadline = received + timeout if isinstance(timeout, (int, float)) and timeout > 0 else None
        if self.in_progress >= self.max_queue:
            logger.warning("Overloaded: %d requests in progress, refusing %s", self.in_progress, request_id)
            return self._error(request_id, "overloaded")
        self.in_progress += 1
        try:
            async with self._workers:
                now = self._clock()
                if deadline is not None and now >= deadline:
                    logger.info("Dropping expired request %s (waited %.2fs)", request_id, now - received)
                    return self._error(request_id, "expired")
                agent = self._choose_agent(deadline, now)
                observation = Observation.from_dict(message["observation"])
                started = self._clock()
                action = await asyncio.to_thread(agent.act, observation)
                if agent is self.agent:
                    elapsed = self._clock() - started
                    self._primary_latency += LATENCY_SMOOTHING * (elapsed - self._primary_latency)
            return {"type": "action", "request_id": request_id, "action": action.to_dict(), "agent": agent.name}
        except Exception as exc:  # report instead of crashing; the server falls back
            logger.exception("Failed to decide request %s", request_id)
            return self._error(request_id, str(exc) or type(exc).__name__)
        finally:
            self.in_progress -= 1
    def _choose_agent(self, deadline: float | None, now: float) -> Agent:
        """The primary agent, or the cheap fallback when the service is busy."""
        if self.fallback is None:
            return self.agent
        waiting = self.in_progress - 1  # requests besides this one
        if waiting >= self.degrade_queue:
            logger.info("Busy (%d waiting): using %s agent", waiting, self.fallback.name)
            return self.fallback
        if deadline is not None and self._primary_latency > (deadline - now) / 2:
            logger.info("Slow (%.2fs per decision): using %s agent", self._primary_latency, self.fallback.name)
            return self.fallback
        return self.agent
    @staticmethod
    def _error(request_id: Any, message: str) -> dict[str, Any]:
        return {"type": "error", "request_id": request_id, "message": message}
@@ -23,4 +23,8 @@ EXPOSE 8083
 HEALTHCHECK --interval=30s --timeout=3s --start-period=5s \
    CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8083/health', timeout=2)"
-CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8083", "--proxy-headers", "--forwarded-allow-ips", "*"]
+# Transport limits as a last line of defence (the app enforces friendlier limits first):
 # messages are < 1 KB, so 64 KB is plenty; at most 1000 concurrent connections.
 CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8083", \
     "--proxy-headers", "--forwarded-allow-ips", "*", \
     "--ws-max-size", "65536", "--ws-max-queue", "16", "--limit-concurrency", "1000"]
@@ -8,11 +8,19 @@ FastAPI game server. It manages sessions, lobbies and games, and talks to the AI
 - `app/protocol.py` — client message schema
 - `app/service.py` — dispatches client messages, reconnects sessions, cleanup
 - `app/lobby.py` — lobbies, seats, host actions, running games and AI turns
- `app/ai_bridge.py` — request/response protocol with the AI service, random-move fallback
+- `app/ai_bridge.py` — request/response protocol with the AI service, random-move fallback,
  in-flight cap and circuit breaker
 - `app/sessions.py` — sessions (token based, survive reloads) and ordered websocket sending
 - `app/limits.py` — token bucket, per-IP rate limiter, connection limiter (abuse protection)
 - `app/load.py` — load monitor (event loop lag, CPU, cgroup throttling) with hysteresis; the
  service refuses new lobbies/games while overloaded or at `MAX_ACTIVE_GAMES`
-Configuration (environment): `AI_SHARED_SECRET`, `AI_DECISION_TIMEOUT` (5s), `AI_MOVE_DELAY` (0.8s),
+Configuration (environment, all in `app/config.py`): `AI_SHARED_SECRET`, `AI_DECISION_TIMEOUT` (5s),
-`TRICK_PAUSE` (1.8s), `SESSION_TTL` (1800s), `MAX_LOBBIES` (1000), `ALLOWED_ORIGINS`, `LOG_LEVEL`.
+`AI_MOVE_DELAY` (0.8s), `TRICK_PAUSE` (1.8s), `SESSION_TTL` (1800s), `MAX_LOBBIES` (1000),
 `ALLOWED_ORIGINS`, `LOG_LEVEL`, and the limits: `MAX_CONNECTIONS` (500), `MAX_CONNECTIONS_PER_IP` (10),
 `MESSAGE_RATE`/`MESSAGE_BURST` (10/30), `LOBBY_CREATE_RATE` (10/min), `MAX_ACTIVE_GAMES` (50),
 `LOAD_*` thresholds, `AI_MAX_INFLIGHT` (64), `AI_BREAKER_FAILURES`/`AI_BREAKER_COOLDOWN` (5/30s).
 See `../docs/04-server.md` and `../docs/09-limits-and-performance.md`.
 ```bash
 pip install -r requirements-dev.txt
@@ -4,14 +4,26 @@ The AI service (``wizard_ai``) connects to ``/internal/ai`` via websocket.
 Whenever an AI seat has to act, the server sends it the seat's observation
 and waits for an action:
-    server -> ai:  {"type": "decide", "request_id": "...", "observation": {...}}
+    server -> ai:  {"type": "decide", "request_id": "...", "timeout": 5.0, "observation": {...}}
-    ai -> server:  {"type": "action", "request_id": "...", "action": {...}}
+    ai -> server:  {"type": "action", "request_id": "...", "action": {...}, "agent": "heuristic"}
    ai -> server:  {"type": "error",  "request_id": "...", "message": "..."}
-Several AI service instances may connect; requests are distributed round
+``timeout`` tells the AI service how long the server waits, so it can drop
-robin. If no instance is connected, it does not answer in time, or it answers
+requests that are already too old. Several AI service instances may connect;
-with an illegal action, a random legal action is played instead so a game
+requests are distributed round robin.
-never gets stuck.
+
 A random legal action is played instead (so a game never gets stuck) when no
 instance is connected, it does not answer in time, it answers with an error or
 an illegal action, too many requests are already waiting for it
 (``max_inflight``), or the circuit breaker is open.
 The circuit breaker protects against a *broken or hanging* AI service: after
 ``breaker_failures`` real failures in a row (timeouts, lost connection,
 malformed or illegal answers, unexpected errors) the AI is not asked for
 ``breaker_cooldown`` seconds, so games don't wait ``timeout`` seconds for every
 AI move. Deliberate load shedding is *not* a failure and does not trip it: the
 in-flight cap, and the AI answering "overloaded" or "expired" (it is healthy
 and answered immediately).
 """
 from __future__ import annotations
@@ -30,6 +42,9 @@ from wizard_game import Action, Observation, action_from_dict
 logger = logging.getLogger(__name__)
 # Errors with which a healthy AI service sheds load (see wizard_ai/service/client.py).
 LOAD_SHEDDING_ERRORS = ("overloaded", "expired")
 class AIConnection:
    def __init__(self, websocket: WebSocket, name: str) -> None:
@@ -55,6 +70,9 @@ class AIBridge:
        timeout: float,
        rng: random.Random | None = None,
        on_change: Callable[[], None] | None = None,
        max_inflight: int = 32,
        breaker_failures: int = 5,
        breaker_cooldown: float = 30.0,
    ) -> None:
        self.timeout = timeout
        # Called whenever an AI service connects or disconnects.
@@ -62,6 +80,11 @@ class AIBridge:
        self._connections: list[AIConnection] = []
        self._round_robin = itertools.count()
        self._rng = rng or random.Random()
        self._max_inflight = max_inflight
        self._breaker_failures = breaker_failures
        self._breaker_cooldown = breaker_cooldown
        self._consecutive_failures = 0
        self._breaker_open_until = 0.0
    @property
    def available(self) -> bool:
@@ -76,32 +99,53 @@ class AIBridge:
        connection = self._pick_connection()
        if connection is None:
            return self._fallback(observation, "no AI service connected", logging.DEBUG)
        loop = asyncio.get_running_loop()
        if loop.time() < self._breaker_open_until:
            return self._fallback(observation, "AI paused after repeated failures", logging.DEBUG)
        if len(connection.pending) >= self._max_inflight:
            return self._fallback(observation, f"{len(connection.pending)} AI requests already waiting")
        request_id = uuid.uuid4().hex
-        future: asyncio.Future[dict[str, Any]] = asyncio.get_running_loop().create_future()
+        future: asyncio.Future[dict[str, Any]] = loop.create_future()
        connection.pending[request_id] = future
        try:
            await connection.send({
                "type": "decide",
                "request_id": request_id,
                "timeout": self.timeout,
                "observation": observation.to_dict(),
            })
            reply = await asyncio.wait_for(future, self.timeout)
        except (TimeoutError, ConnectionError, RuntimeError, WebSocketDisconnect) as exc:
-            return self._fallback(observation, f"AI request failed: {exc!r}")
+            return self._failure(observation, f"AI request failed: {exc!r}")
        finally:
            connection.pending.pop(request_id, None)
        if reply.get("type") != "action":
-            return self._fallback(observation, f"AI error: {reply.get('message')}")
+            message = reply.get("message")
            if message in LOAD_SHEDDING_ERRORS:
                self._consecutive_failures = 0  # the service is alive and protecting itself
                return self._fallback(observation, f"AI service {message}")
            return self._failure(observation, f"AI error: {message}")
        try:
            action = action_from_dict(reply.get("action", {}))
        except ValueError as exc:
-            return self._fallback(observation, f"malformed AI action: {exc}")
+            return self._failure(observation, f"malformed AI action: {exc}")
        if action not in legal:
-            return self._fallback(observation, f"illegal AI action {action}")
+            return self._failure(observation, f"illegal AI action {action}")
        self._consecutive_failures = 0
        if reply.get("agent"):
            logger.debug("AI decision by %s agent", reply["agent"])
        return action
    def stats(self) -> dict[str, Any]:
        loop_time = asyncio.get_running_loop().time()
        return {
            "connections": len(self._connections),
            "pending": sum(len(c.pending) for c in self._connections),
            "paused": loop_time < self._breaker_open_until,
        }
    async def serve(self, websocket: WebSocket) -> None:
        """Handle one AI service connection until it disconnects."""
        connection = AIConnection(websocket, name=websocket.client.host if websocket.client else "?")
@@ -136,6 +180,16 @@ class AIBridge:
            return None
        return self._connections[next(self._round_robin) % len(self._connections)]
    def _failure(self, observation: Observation, reason: str) -> Action:
        """A failed AI request: count it for the circuit breaker, then fall back."""
        self._consecutive_failures += 1
        if self._consecutive_failures >= self._breaker_failures:
            self._consecutive_failures = 0
            self._breaker_open_until = asyncio.get_running_loop().time() + self._breaker_cooldown
            logger.warning("AI failed %d times in a row, pausing AI requests for %.0fs",
                           self._breaker_failures, self._breaker_cooldown)
        return self._fallback(observation, reason)
    def _fallback(self, observation: Observation, reason: str, level: int = logging.WARNING) -> Action:
        logger.log(level, "Playing random move for player %d: %s", observation.player, reason)
        return self._rng.choice(observation.legal_actions)
@@ -35,5 +35,46 @@ class Settings:
    )
    log_level: str = os.environ.get("LOG_LEVEL", "INFO")
    # --- Abuse limits -------------------------------------------------------
    # Open websocket connections in total and per client IP.
    max_connections: int = _int("MAX_CONNECTIONS", 500)
    max_connections_per_ip: int = _int("MAX_CONNECTIONS_PER_IP", 10)
    # Messages per second a single connection may send (token bucket).
    message_rate: float = _float("MESSAGE_RATE", 10.0)
    message_burst: int = _int("MESSAGE_BURST", 30)
    # Dropped messages after which a flooding connection is closed.
    max_rate_violations: int = _int("MAX_RATE_VIOLATIONS", 50)
    # Lobbies a single IP may create per minute.
    lobby_create_rate: int = _int("LOBBY_CREATE_RATE", 10)
    # --- Load control -------------------------------------------------------
    # Games running at the same time; further game starts are refused.
    max_active_games: int = _int("MAX_ACTIVE_GAMES", 50)
    load_sample_interval: float = _float("LOAD_SAMPLE_INTERVAL", 1.0)
    # Event loop lag (seconds) / CPU share of the available quota (0..1).
    load_lag_busy: float = _float("LOAD_LAG_BUSY", 0.1)
    load_lag_overload: float = _float("LOAD_LAG_OVERLOAD", 0.25)
    # A single sample with this much lag means overload right away: when the
    # server is that slow, even the load sampler runs rarely.
    load_lag_severe: float = _float("LOAD_LAG_SEVERE", 1.0)
    load_cpu_busy: float = _float("LOAD_CPU_BUSY", 0.7)
    load_cpu_overload: float = _float("LOAD_CPU_OVERLOAD", 0.9)
    # Share of CPU periods in which the container was throttled (cgroup v2).
    load_throttle_overload: float = _float("LOAD_THROTTLE_OVERLOAD", 0.5)
    # Consecutive overloaded samples to enter / good samples to leave overload.
    load_overload_samples: int = _int("LOAD_OVERLOAD_SAMPLES", 3)
    load_recover_samples: int = _int("LOAD_RECOVER_SAMPLES", 10)
    # "normal", "busy" or "overloaded" to pin the level (testing only).
    load_force_level: str = os.environ.get("LOAD_FORCE_LEVEL", "")
    # --- AI service protection ----------------------------------------------
    # Unanswered decide requests per AI connection before falling back.
    # Each running game has at most one AI deciding at a time, so keep this at
    # least MAX_ACTIVE_GAMES; 64 matches the AI service's AI_MAX_QUEUE.
    ai_max_inflight: int = _int("AI_MAX_INFLIGHT", 64)
    # Consecutive AI failures that pause AI requests for ai_breaker_cooldown seconds.
    ai_breaker_failures: int = _int("AI_BREAKER_FAILURES", 5)
    ai_breaker_cooldown: float = _float("AI_BREAKER_COOLDOWN", 30.0)
 settings = Settings()
@@ -0,0 +1,90 @@
 """Rate and connection limits that protect the server from floods and abuse."""
 from __future__ import annotations
 import time
 from collections import Counter
 from collections.abc import Callable, Hashable
 Clock = Callable[[], float]
 class TokenBucket:
    """Allows ``rate`` actions per second on average with bursts up to ``burst``."""
    def __init__(self, rate: float, burst: float, clock: Clock = time.monotonic) -> None:
        self.rate = rate
        self.burst = burst
        self._clock = clock
        self._tokens = float(burst)
        self._updated = clock()
    def allow(self, cost: float = 1.0) -> bool:
        self._refill()
        if self._tokens >= cost:
            self._tokens -= cost
            return True
        return False
    @property
    def full(self) -> bool:
        self._refill()
        return self._tokens >= self.burst
    def _refill(self) -> None:
        now = self._clock()
        self._tokens = min(self.burst, self._tokens + (now - self._updated) * self.rate)
        self._updated = now
 class KeyedRateLimiter:
    """One token bucket per key (e.g. per client IP)."""
    def __init__(self, rate: float, burst: float, clock: Clock = time.monotonic) -> None:
        self.rate = rate
        self.burst = burst
        self._clock = clock
        self._buckets: dict[Hashable, TokenBucket] = {}
    def allow(self, key: Hashable) -> bool:
        bucket = self._buckets.get(key)
        if bucket is None:
            bucket = self._buckets[key] = TokenBucket(self.rate, self.burst, self._clock)
        return bucket.allow()
    def prune(self) -> None:
        """Forget keys whose bucket has refilled completely (they behave like new keys)."""
        for key in [key for key, bucket in self._buckets.items() if bucket.full]:
            del self._buckets[key]
    def __len__(self) -> int:
        return len(self._buckets)
 class ConnectionLimiter:
    """Counts open connections in total and per client IP."""
    def __init__(self, max_total: int, max_per_ip: int) -> None:
        self.max_total = max_total
        self.max_per_ip = max_per_ip
        self._per_ip: Counter[str] = Counter()
        self._total = 0
    @property
    def total(self) -> int:
        return self._total
    def acquire(self, ip: str) -> bool:
        if self._total >= self.max_total or self._per_ip[ip] >= self.max_per_ip:
            return False
        self._per_ip[ip] += 1
        self._total += 1
        return True
    def release(self, ip: str) -> None:
        if self._per_ip[ip] <= 0:
            return
        self._per_ip[ip] -= 1
        self._total -= 1
        if self._per_ip[ip] == 0:
            del self._per_ip[ip]
@@ -0,0 +1,248 @@
 """Load monitoring: detects when the server runs out of compute.
 Every ``load_sample_interval`` seconds the monitor measures
 - **event loop lag**: how much later than scheduled the loop woke up. This is
  the most direct signal that the server is slow for players;
 - **CPU usage** of the process relative to what it may use (the container's
  CPU quota, at most one core because Python runs game logic on one core);
 - **CPU throttling** from cgroup v2 (``/sys/fs/cgroup/cpu.stat``): the share
  of scheduling periods in which the container hit its CPU limit.
 Each sample is classified as normal / busy / overloaded, and ``next_level``
 turns the recent classifications into the current level with hysteresis, so a
 single slow second does not flip the state back and forth.
 """
 from __future__ import annotations
 import asyncio
 import logging
 import time
 from collections import deque
 from collections.abc import Callable, Sequence
 from dataclasses import dataclass
 from enum import Enum
 from pathlib import Path
 from .config import Settings
 logger = logging.getLogger(__name__)
 CGROUP_DIR = Path("/sys/fs/cgroup")
 class LoadLevel(str, Enum):
    NORMAL = "normal"
    BUSY = "busy"
    OVERLOADED = "overloaded"
@dataclass(frozen=True, slots=True)
 class LoadSample:
    loop_lag: float
    # Share of the available CPU used (None if unknown).
    cpu: float | None = None
    # Share of CPU periods in which the container was throttled (None if unknown).
    throttled: float | None = None
@dataclass(frozen=True, slots=True)
 class LoadThresholds:
    lag_busy: float
    lag_overload: float
    lag_severe: float
    cpu_busy: float
    cpu_overload: float
    throttle_overload: float
    overload_samples: int
    recover_samples: int
    @classmethod
    def from_settings(cls, settings: Settings) -> LoadThresholds:
        return cls(
            lag_busy=settings.load_lag_busy,
            lag_overload=settings.load_lag_overload,
            lag_severe=settings.load_lag_severe,
            cpu_busy=settings.load_cpu_busy,
            cpu_overload=settings.load_cpu_overload,
            throttle_overload=settings.load_throttle_overload,
            overload_samples=max(1, settings.load_overload_samples),
            recover_samples=max(1, settings.load_recover_samples),
        )
 def classify(sample: LoadSample, t: LoadThresholds) -> LoadLevel:
    """Level of a single sample."""
    if (
        sample.loop_lag >= t.lag_overload
        or (sample.cpu is not None and sample.cpu >= t.cpu_overload)
        or (sample.throttled is not None and sample.throttled >= t.throttle_overload)
    ):
        return LoadLevel.OVERLOADED
    if sample.loop_lag >= t.lag_busy or (sample.cpu is not None and sample.cpu >= t.cpu_busy):
        return LoadLevel.BUSY
    return LoadLevel.NORMAL
 def next_level(
    current: LoadLevel, recent: Sequence[LoadLevel], t: LoadThresholds, severe: bool = False
 ) -> LoadLevel:
    """Current level from the recent sample levels (oldest first), with hysteresis.
    - Overloaded after ``overload_samples`` overloaded samples in a row, or at
      once when the latest sample was ``severe`` (extreme event loop lag).
    - Leaving overload (and busy) requires ``recover_samples`` samples in a row
      below that level.
    - Busy is entered immediately (it is only a hint for players).
    """
    if severe:
        return LoadLevel.OVERLOADED
    if not recent:
        return current
    last_bad = recent[-t.overload_samples:]
    if len(last_bad) == t.overload_samples and all(level is LoadLevel.OVERLOADED for level in last_bad):
        return LoadLevel.OVERLOADED
    window = recent[-t.recover_samples:]
    full_window = len(window) == t.recover_samples
    if current is LoadLevel.OVERLOADED and not (
        full_window and all(level is not LoadLevel.OVERLOADED for level in window)
    ):
        return LoadLevel.OVERLOADED
    if recent[-1] is not LoadLevel.NORMAL:
        return LoadLevel.BUSY
    if current is not LoadLevel.NORMAL and not (full_window and all(level is LoadLevel.NORMAL for level in window)):
        return LoadLevel.BUSY
    return LoadLevel.NORMAL
 # --- cgroup v2 helpers ------------------------------------------------------
 def parse_cpu_max(text: str) -> float | None:
    """CPU quota in cores from ``cpu.max`` ("max 100000" = unlimited)."""
    parts = text.split()
    if len(parts) != 2 or parts[0] == "max":
        return None
    try:
        return int(parts[0]) / int(parts[1])
    except (ValueError, ZeroDivisionError):
        return None
 def parse_cpu_stat(text: str) -> dict[str, int]:
    stats: dict[str, int] = {}
    for line in text.splitlines():
        key, _, value = line.partition(" ")
        if value.strip().isdigit():
            stats[key] = int(value)
    return stats
 def own_cgroup_dir(root: Path = CGROUP_DIR, proc_cgroup: Path = Path("/proc/self/cgroup")) -> Path:
    """The cgroup v2 directory of this process.
    In a container this is ``/sys/fs/cgroup`` itself (cgroup namespace); on a
    host it is a sub directory such as ``/sys/fs/cgroup/user.slice/…scope``.
    """
    for line in (_read(proc_cgroup) or "").splitlines():
        if line.startswith("0::"):
            candidate = root / line[3:].strip().lstrip("/")
            if candidate.is_dir():
                return candidate
    return root
 def _read(path: Path) -> str | None:
    try:
        return path.read_text()
    except OSError:
        return None
 class LoadMonitor:
    def __init__(
        self,
        settings: Settings,
        on_change: Callable[[LoadLevel], None] | None = None,
        cgroup_dir: Path | None = None,
    ) -> None:
        self._settings = settings
        self._thresholds = LoadThresholds.from_settings(settings)
        self._on_change = on_change
        self._cgroup_dir = cgroup_dir if cgroup_dir is not None else own_cgroup_dir()
        self._forced = LoadLevel(settings.load_force_level) if settings.load_force_level else None
        self._level = LoadLevel.NORMAL
        self._recent: deque[LoadLevel] = deque(
            maxlen=max(self._thresholds.overload_samples, self._thresholds.recover_samples)
        )
        self.last_sample = LoadSample(loop_lag=0.0)
        quota = parse_cpu_max(_read(self._cgroup_dir / "cpu.max") or "")
        # Game logic runs on one core, so more than one core cannot be used.
        self._cpu_capacity = min(quota, 1.0) if quota else 1.0
        self._last_cpu_time = time.process_time()
        self._last_wall = time.monotonic()
        self._last_cgroup = self._cgroup_stats()
    @property
    def level(self) -> LoadLevel:
        return self._forced or self._level
    def record(self, sample: LoadSample) -> None:
        """Add a sample and update the level (called by ``run``; public for tests)."""
        self.last_sample = sample
        self._recent.append(classify(sample, self._thresholds))
        severe = sample.loop_lag >= self._thresholds.lag_severe
        new_level = next_level(self._level, list(self._recent), self._thresholds, severe)
        if new_level is self._level:
            return
        logger.log(
            logging.WARNING if new_level is LoadLevel.OVERLOADED else logging.INFO,
            "Load level %s -> %s (lag %.3fs, cpu %s, throttled %s)",
            self._level.value, new_level.value, sample.loop_lag, sample.cpu, sample.throttled,
        )
        self._level = new_level
        if self._on_change is not None and self._forced is None:
            self._on_change(new_level)
    async def run(self) -> None:
        loop = asyncio.get_running_loop()
        interval = self._settings.load_sample_interval
        while True:
            started = loop.time()
            await asyncio.sleep(interval)
            lag = max(0.0, loop.time() - started - interval)
            try:
                self.record(self._measure(lag))
            except Exception:
                logger.exception("Load measurement failed")
    def stats(self) -> dict[str, object]:
        sample = self.last_sample
        return {
            "load": self.level.value,
            "loop_lag": round(sample.loop_lag, 4),
            "cpu": None if sample.cpu is None else round(sample.cpu, 3),
            "throttled": None if sample.throttled is None else round(sample.throttled, 3),
        }
    def _measure(self, lag: float) -> LoadSample:
        now_cpu, now_wall = time.process_time(), time.monotonic()
        wall = now_wall - self._last_wall
        cpu = (now_cpu - self._last_cpu_time) / wall / self._cpu_capacity if wall > 0 else None
        self._last_cpu_time, self._last_wall = now_cpu, now_wall
        throttled = None
        stats = self._cgroup_stats()
        if stats and self._last_cgroup:
            periods = stats.get("nr_periods", 0) - self._last_cgroup.get("nr_periods", 0)
            if periods > 0:
                throttled = (stats.get("nr_throttled", 0) - self._last_cgroup.get("nr_throttled", 0)) / periods
        self._last_cgroup = stats
        return LoadSample(loop_lag=lag, cpu=cpu, throttled=throttled)
    def _cgroup_stats(self) -> dict[str, int]:
        text = _read(self._cgroup_dir / "cpu.stat")
        return parse_cpu_stat(text) if text else {}
@@ -5,7 +5,7 @@ from __future__ import annotations
 import asyncio
 import logging
 import secrets
-from collections.abc import Iterable
+from collections.abc import Callable, Iterable
 from dataclasses import dataclass
 from enum import Enum
 from typing import Any
@@ -24,6 +24,7 @@ from wizard_game import (
 from .ai_bridge import AIBridge
 from .config import Settings
 from .protocol import ERROR_BUSY
 from .sessions import Session
 logger = logging.getLogger(__name__)
@@ -40,6 +41,11 @@ AI_NAMES = (
 class LobbyError(Exception):
    """An error that is reported to the client that caused it."""
    def __init__(self, message: str, code: str | None = None) -> None:
        super().__init__(message)
        # Optional machine readable code, see ``protocol.ERROR_*``.
        self.code = code
 class LobbyStatus(str, Enum):
    WAITING = "waiting"
@@ -60,8 +66,16 @@ class Seat:
 class Lobby:
-    def __init__(self, code: str, bridge: AIBridge, settings: Settings) -> None:
+    def __init__(
        self,
        code: str,
        bridge: AIBridge,
        settings: Settings,
        on_status_change: Callable[[], None] | None = None,
    ) -> None:
        self.code = code
        # Called when a game starts or ends (the number of running games changes).
        self._on_status_change = on_status_change
        self.seats: list[Seat] = []
        self.host_id: str | None = None
        self.options = GameOptions()
@@ -188,6 +202,7 @@ class Lobby:
        self.status = LobbyStatus.PLAYING
        self._pause_until = 0.0
        logger.info("Lobby %s: game started with %d players", self.code, len(self.seats))
        self._notify_status_change()
        self._schedule_ai()
    def return_to_lobby(self, by: Session) -> None:
@@ -224,6 +239,9 @@ class Lobby:
        self._closed = True
        if self._ai_task and not self._ai_task.done():
            self._ai_task.cancel()
        if self.status is LobbyStatus.PLAYING:
            self.status = LobbyStatus.FINISHED
            self._notify_status_change()
    # --- Internals ----------------------------------------------------------
@@ -247,12 +265,19 @@ class Lobby:
        loop = asyncio.get_running_loop()
        if any(isinstance(e, TrickCompleted) for e in events):
            self._pause_until = loop.time() + self._settings.trick_pause
-        if any(isinstance(e, GameFinished) for e in events):
+        finished = any(isinstance(e, GameFinished) for e in events)
        if finished:
            self.status = LobbyStatus.FINISHED
            logger.info("Lobby %s: game finished", self.code)
        self.broadcast(events)
        if finished:
            self._notify_status_change()
        self._schedule_ai()
    def _notify_status_change(self) -> None:
        if self._on_status_change is not None:
            self._on_status_change()
    def _current_ai_seat(self) -> int | None:
        game = self.game
        if self._closed or self.status is not LobbyStatus.PLAYING or game is None:
@@ -292,9 +317,15 @@ class Lobby:
 class LobbyManager:
-    def __init__(self, bridge: AIBridge, settings: Settings) -> None:
+    def __init__(
        self,
        bridge: AIBridge,
        settings: Settings,
        on_games_changed: Callable[[], None] | None = None,
    ) -> None:
        self._bridge = bridge
        self._settings = settings
        self._on_games_changed = on_games_changed
        self._lobbies: dict[str, Lobby] = {}
    def __len__(self) -> int:
@@ -303,14 +334,17 @@ class LobbyManager:
    def __iter__(self):
        return iter(list(self._lobbies.values()))
    def active_games(self) -> int:
        return sum(lobby.status is LobbyStatus.PLAYING for lobby in self._lobbies.values())
    def create(self) -> Lobby:
        if len(self._lobbies) >= self._settings.max_lobbies:
-            raise LobbyError("Too many open lobbies, please try again later")
+            raise LobbyError("Too many open lobbies, please try again later", ERROR_BUSY)
        while True:
            code = "".join(secrets.choice(LOBBY_CODE_ALPHABET) for _ in range(LOBBY_CODE_LENGTH))
            if code not in self._lobbies:
                break
-        lobby = Lobby(code, self._bridge, self._settings)
+        lobby = Lobby(code, self._bridge, self._settings, self._on_games_changed)
        self._lobbies[code] = lobby
        logger.info("Lobby %s created (%d open)", code, len(self._lobbies))
        return lobby
@@ -30,23 +30,22 @@ def create_app(config: Settings = settings) -> FastAPI:
    async def lifespan(_: FastAPI) -> AsyncIterator[None]:
        if not config.ai_shared_secret:
            logger.warning("AI_SHARED_SECRET is not set, /internal/ai is unauthenticated")
-        cleanup = asyncio.create_task(service.cleanup_loop())
+        tasks = [
            asyncio.create_task(service.cleanup_loop()),
            asyncio.create_task(service.load.run()),
        ]
        try:
            yield
        finally:
-            cleanup.cancel()
+            for task in tasks:
                task.cancel()
    app = FastAPI(title="Wizard game server", lifespan=lifespan, docs_url=None, redoc_url=None)
    app.state.service = service
    @app.get("/health")
    async def health() -> dict:
-        return {
+        return service.health()
            "status": "ok",
            "lobbies": len(service.lobbies),
            "sessions": len(service.sessions),
            "ai_available": service.bridge.available,
        }
    @app.websocket("/ws")
    async def client_socket(websocket: WebSocket) -> None:
@@ -5,7 +5,8 @@ messages are plain dicts:
 - ``{"type": "welcome", "player_id", "token", "name"}``
 - ``{"type": "state", "lobby": LobbyView | null, "game": GameView | null, "events": [...]}``
- ``{"type": "error", "message"}``
+- ``{"type": "error", "message", "code"?}``
 - ``{"type": "server_status", "load", "accepting_games"}``
 """
 from __future__ import annotations
@@ -75,5 +76,16 @@ ClientMessage = Annotated[
 client_message_adapter: TypeAdapter[ClientMessage] = TypeAdapter(ClientMessage)
-def error(message: str) -> dict:
+# Machine readable error codes (the frontend reacts to them).
-    return {"type": "error", "message": message}
+ERROR_BUSY = "busy"                  # server has no capacity for this right now
 ERROR_RATE_LIMITED = "rate_limited"  # client sends too many messages / creates too many lobbies
 # Websocket close code "Try Again Later" (RFC 6455 registry), used when full.
 CLOSE_TRY_AGAIN_LATER = 1013
 def error(message: str, code: str | None = None) -> dict:
    message_dict: dict = {"type": "error", "message": message}
    if code:
        message_dict["code"] = code
    return message_dict
@@ -15,6 +15,8 @@ from wizard_game import GameOptions
 from . import protocol
 from .ai_bridge import AIBridge
 from .config import Settings
 from .limits import ConnectionLimiter, KeyedRateLimiter, TokenBucket
 from .load import LoadLevel, LoadMonitor
 from .lobby import Lobby, LobbyError, LobbyManager
 from .sessions import Connection, Session, SessionRegistry
@@ -27,8 +29,53 @@ class WizardService:
    def __init__(self, settings: Settings) -> None:
        self.settings = settings
        self.sessions = SessionRegistry()
-        self.bridge = AIBridge(settings.ai_decision_timeout, on_change=self._on_ai_availability_changed)
+        self.bridge = AIBridge(
-        self.lobbies = LobbyManager(self.bridge, settings)
+            settings.ai_decision_timeout,
            on_change=self._on_ai_availability_changed,
            max_inflight=settings.ai_max_inflight,
            breaker_failures=settings.ai_breaker_failures,
            breaker_cooldown=settings.ai_breaker_cooldown,
        )
        self.lobbies = LobbyManager(self.bridge, settings, on_games_changed=self._broadcast_server_status)
        self.connections = ConnectionLimiter(settings.max_connections, settings.max_connections_per_ip)
        self.lobby_creation = KeyedRateLimiter(settings.lobby_create_rate / 60, settings.lobby_create_rate)
        self.load = LoadMonitor(settings, on_change=lambda _level: self._broadcast_server_status())
        self._last_server_status: dict | None = None
    # --- Capacity -----------------------------------------------------------
    @property
    def accepting_games(self) -> bool:
        """Whether new lobbies may be created and games started.
        Running games are never affected; only new work is refused.
        """
        return (
            self.load.level is not LoadLevel.OVERLOADED
            and self.lobbies.active_games() < self.settings.max_active_games
        )
    def server_status(self) -> dict:
        return {
            "type": "server_status",
            "load": self.load.level.value,
            "accepting_games": self.accepting_games,
        }
    def _broadcast_server_status(self) -> None:
        status = self.server_status()
        if status == self._last_server_status:
            return
        self._last_server_status = status
        for session in self.sessions.connected():
            session.send(status)
    def _require_capacity(self) -> None:
        if not self.accepting_games:
            raise LobbyError(
                "The server is busy right now, new games can't be started. Please try again in a moment.",
                protocol.ERROR_BUSY,
            )
    # --- Client connections -------------------------------------------------
@@ -41,10 +88,26 @@ class WizardService:
        await websocket.accept()
        connection = Connection(websocket)
        connection.start()
        if not self.connections.acquire(connection.ip):
            logger.warning("Refusing connection from %s: connection limit reached", connection.ip)
            connection.send(protocol.error("The server is full right now, please try again later.", protocol.ERROR_BUSY))
            await connection.close(code=protocol.CLOSE_TRY_AGAIN_LATER)
            return
        bucket = TokenBucket(self.settings.message_rate, self.settings.message_burst)
        violations = 0
        session: Session | None = None
        try:
            while True:
                raw = await websocket.receive_text()
                if not bucket.allow():
                    violations += 1
                    if violations >= self.settings.max_rate_violations:
                        logger.warning("Closing connection from %s: message flood", connection.ip)
                        break
                    if violations == 1 or violations % 10 == 0:
                        connection.send(protocol.error("Too many messages, slow down.", protocol.ERROR_RATE_LIMITED))
                    continue
                try:
                    message = protocol.client_message_adapter.validate_json(raw)
                except ValidationError as exc:
@@ -60,16 +123,17 @@ class WizardService:
                try:
                    self._dispatch(session, message)
                except LobbyError as exc:
-                    session.send(protocol.error(str(exc)))
+                    session.send(protocol.error(str(exc), exc.code))
        except WebSocketDisconnect:
            pass
        finally:
            self.connections.release(connection.ip)
            if session is not None and session.connection is connection:
                session.connection = None
                session.last_seen = time.monotonic()
                if lobby := self.lobbies.get(session.lobby_code):
                    lobby.broadcast()
-            await connection.close()
+            await connection.close(code=1008 if violations >= self.settings.max_rate_violations else 1000)
    def _hello(self, connection: Connection, token: str | None) -> Session:
        session = self.sessions.get(token)
@@ -87,6 +151,7 @@ class WizardService:
            "token": session.token,
            "name": session.name,
        })
        session.send(self.server_status())
        lobby = self.lobbies.get(session.lobby_code)
        if lobby is not None and lobby.seat_index(session.player_id) is not None:
            lobby.add_human(session)  # re-binds the seat to this session
@@ -99,6 +164,11 @@ class WizardService:
    def _dispatch(self, session: Session, message: object) -> None:
        match message:
            case protocol.CreateLobby(name=name):
                self._require_capacity()
                ip = session.connection.ip if session.connection else "unknown"
                if not self.lobby_creation.allow(ip):
                    raise LobbyError("You are creating lobbies too quickly, please wait a moment.",
                                     protocol.ERROR_RATE_LIMITED)
                self._leave_current_lobby(session)
                session.name = name
                lobby = self.lobbies.create()
@@ -132,6 +202,7 @@ class WizardService:
                lobby.broadcast()
            case protocol.StartGame():
                lobby = self._lobby_of(session)
                self._require_capacity()
                lobby.start(session)
                lobby.broadcast()
            case protocol.ReturnToLobby():
@@ -200,6 +271,20 @@ class WizardService:
        for session in self.sessions.expired(self.settings.session_ttl):
            self._leave_current_lobby(session)
            self.sessions.remove(session)
        self.lobby_creation.prune()
    def health(self) -> dict:
        return {
            "status": "ok",  # stays ok under load: an overloaded server must not be restarted
            **self.load.stats(),
            "accepting_games": self.accepting_games,
            "lobbies": len(self.lobbies),
            "active_games": self.lobbies.active_games(),
            "sessions": len(self.sessions),
            "connections": self.connections.total,
            "ai_available": self.bridge.available,
            "ai": self.bridge.stats(),
        }
 def _validation_message(exc: ValidationError) -> str:
@@ -24,6 +24,8 @@ logger = logging.getLogger(__name__)
 class Connection:
    def __init__(self, websocket: WebSocket) -> None:
        self.websocket = websocket
        # Client IP; behind the reverse proxy uvicorn takes it from X-Forwarded-For.
        self.ip = websocket.client.host if websocket.client else "unknown"
        self._queue: asyncio.Queue[dict[str, Any] | None] = asyncio.Queue()
        self._writer: asyncio.Task | None = None
@@ -85,6 +87,9 @@ class SessionRegistry:
    def remove(self, session: Session) -> None:
        self._by_token.pop(session.token, None)
    def connected(self) -> list[Session]:
        return [s for s in self._by_token.values() if s.connected]
    def expired(self, ttl: float) -> list[Session]:
        now = time.monotonic()
        return [s for s in self._by_token.values() if not s.connected and now - s.last_seen > ttl]
@@ -0,0 +1,26 @@
 import contextlib
 import pytest
 from fastapi.testclient import TestClient
 from app.config import Settings
 from app.main import create_app
 from helpers import TEST_SETTINGS
@pytest.fixture
 def make_client():
    """Factory: ``make_client(**settings_overrides)`` returns a started TestClient."""
    with contextlib.ExitStack() as stack:
        def factory(**overrides):
            settings = Settings(**{**TEST_SETTINGS, **overrides})
            test_client = stack.enter_context(TestClient(create_app(settings)))
            # Websocket sessions are closed before the app stops (entered later, exited first).
            test_client.sockets = stack.enter_context(contextlib.ExitStack())
            return test_client
        yield factory
@pytest.fixture
 def client(make_client):
    return make_client()
@@ -0,0 +1,33 @@
 """Helpers shared by the server tests."""
 SECRET = "test-secret"
 # Tests play much faster than humans, so the message rate limit is raised.
 TEST_SETTINGS = dict(
    ai_shared_secret=SECRET,
    ai_move_delay=0,
    trick_pause=0,
    ai_decision_timeout=2,
    message_rate=10_000,
    message_burst=10_000,
 )
 def receive_until(ws, predicate, limit=5000):
    for _ in range(limit):
        message = ws.receive_json()
        if predicate(message):
            return message
    raise AssertionError("expected message not received")
 def is_state(message):
    return message["type"] == "state"
 def connect(client, token=None):
    ws = client.sockets.enter_context(client.websocket_connect("/ws"))
    ws.send_json({"type": "hello", "token": token})
    welcome = receive_until(ws, lambda m: m["type"] == "welcome")
    receive_until(ws, is_state)
    return ws, welcome
@@ -0,0 +1,250 @@
 """Unit tests of the limiter, load-monitor and AI-bridge protection building blocks."""
 import asyncio
 import random
 from wizard_game import WizardGame
 from app.ai_bridge import AIBridge, AIConnection
 from app.config import Settings
 from app.limits import ConnectionLimiter, KeyedRateLimiter, TokenBucket
 from app.load import (
    LoadLevel,
    LoadMonitor,
    LoadSample,
    LoadThresholds,
    classify,
    next_level,
    own_cgroup_dir,
    parse_cpu_max,
    parse_cpu_stat,
 )
 N, B, O = LoadLevel.NORMAL, LoadLevel.BUSY, LoadLevel.OVERLOADED
 class FakeClock:
    def __init__(self):
        self.now = 0.0
    def __call__(self):
        return self.now
 # --- TokenBucket / KeyedRateLimiter / ConnectionLimiter -----------------------
 def test_token_bucket_burst_and_refill():
    clock = FakeClock()
    bucket = TokenBucket(rate=2, burst=3, clock=clock)
    assert [bucket.allow() for _ in range(4)] == [True, True, True, False]
    clock.now = 0.5  # one token back
    assert bucket.allow() and not bucket.allow()
    clock.now = 100  # never more than the burst
    assert sum(bucket.allow() for _ in range(10)) == 3
 def test_keyed_rate_limiter_is_per_key_and_prunes():
    clock = FakeClock()
    limiter = KeyedRateLimiter(rate=1, burst=1, clock=clock)
    assert limiter.allow("a") and not limiter.allow("a")
    assert limiter.allow("b")
    clock.now = 10
    limiter.prune()
    assert len(limiter) == 0
 def test_connection_limiter():
    limiter = ConnectionLimiter(max_total=3, max_per_ip=2)
    assert limiter.acquire("1.1.1.1") and limiter.acquire("1.1.1.1")
    assert not limiter.acquire("1.1.1.1")
    assert limiter.acquire("2.2.2.2")
    assert not limiter.acquire("3.3.3.3")  # total reached
    limiter.release("1.1.1.1")
    assert limiter.total == 2
    assert limiter.acquire("3.3.3.3")
    limiter.release("9.9.9.9")  # unknown ip is ignored
    assert limiter.total == 3
 # --- Load levels --------------------------------------------------------------
 T = LoadThresholds(
    lag_busy=0.1, lag_overload=0.25, lag_severe=1.0, cpu_busy=0.7, cpu_overload=0.9,
    throttle_overload=0.5, overload_samples=3, recover_samples=4,
 )
 def test_classify():
    assert classify(LoadSample(0.0, 0.1, 0.0), T) is N
    assert classify(LoadSample(0.15), T) is B
    assert classify(LoadSample(0.0, 0.75), T) is B
    assert classify(LoadSample(0.3), T) is O
    assert classify(LoadSample(0.0, 0.95), T) is O
    assert classify(LoadSample(0.0, 0.2, 0.6), T) is O
 def run_levels(samples):
    level, recent, levels = N, [], []
    for sample in samples:
        recent.append(sample)
        level = next_level(level, recent, T)
        levels.append(level)
    return levels
 def test_overload_needs_consecutive_samples():
    assert run_levels([O, O, N]) == [B, B, B]
    assert run_levels([O, O, O]) == [B, B, O]
 def test_overload_recovery_needs_a_calm_window():
    levels = run_levels([O, O, O, N, N, N, B, N, N, N, N])
    assert levels[3:6] == [O, O, O]  # still overloaded while recovering
    assert levels[6] is B  # 4 samples without overload -> busy (last sample busy)
    assert levels[-1] is N  # 4 normal samples -> normal
 def test_severe_lag_overloads_immediately():
    assert next_level(N, [O], T, severe=True) is O
    monitor = LoadMonitor(Settings())
    monitor.record(LoadSample(loop_lag=2.0))
    assert monitor.level is O
 def test_busy_is_immediate_and_sticky():
    assert run_levels([N, B, N, N, N, N]) == [N, B, B, B, B, N]
 # --- cgroup parsing / LoadMonitor ---------------------------------------------
 def test_parse_cgroup_files():
    assert parse_cpu_max("max 100000\n") is None
    assert parse_cpu_max("50000 100000\n") == 0.5
    assert parse_cpu_max("") is None
    stats = parse_cpu_stat("usage_usec 10\nnr_periods 200\nnr_throttled 50\n")
    assert stats["nr_periods"] == 200 and stats["nr_throttled"] == 50
 def test_own_cgroup_dir(tmp_path):
    (tmp_path / "user.slice" / "app.scope").mkdir(parents=True)
    proc = tmp_path / "cgroup"
    proc.write_text("0::/user.slice/app.scope\n")
    assert own_cgroup_dir(tmp_path, proc) == tmp_path / "user.slice" / "app.scope"
    proc.write_text("0::/\n")  # inside a container
    assert own_cgroup_dir(tmp_path, proc) == tmp_path
    assert own_cgroup_dir(tmp_path, tmp_path / "missing") == tmp_path
 def test_monitor_reads_cgroup_throttling(tmp_path):
    (tmp_path / "cpu.max").write_text("50000 100000\n")
    (tmp_path / "cpu.stat").write_text("nr_periods 100\nnr_throttled 0\n")
    monitor = LoadMonitor(Settings(), cgroup_dir=tmp_path)
    (tmp_path / "cpu.stat").write_text("nr_periods 200\nnr_throttled 80\n")
    sample = monitor._measure(lag=0.0)
    assert sample.throttled == 0.8
 def test_monitor_levels_and_callback():
    changes = []
    monitor = LoadMonitor(Settings(load_overload_samples=2, load_recover_samples=2), on_change=changes.append)
    for _ in range(2):
        monitor.record(LoadSample(loop_lag=0.5))  # overloaded, but not severe
    assert monitor.level is O
    for _ in range(2):
        monitor.record(LoadSample(loop_lag=0.0))
    assert monitor.level is N
    assert changes == [B, O, N]
    assert monitor.stats()["load"] == "normal"
 def test_forced_level():
    monitor = LoadMonitor(Settings(load_force_level="overloaded"))
    assert monitor.level is O
    monitor.record(LoadSample(loop_lag=0.0))
    assert monitor.level is O
 # --- AI bridge protection -----------------------------------------------------
 class SilentWebSocket:
    """Accepts requests but never answers (a hanging AI service)."""
    def __init__(self):
        self.sent = []
    async def send_json(self, message):
        self.sent.append(message)
 def observation():
    game = WizardGame(3, seed=1)
    return game.observation(game.current_player)
 def test_ai_circuit_breaker():
    async def scenario():
        bridge = AIBridge(timeout=0.01, rng=random.Random(0), breaker_failures=2, breaker_cooldown=60)
        ws = SilentWebSocket()
        bridge._connections.append(AIConnection(ws, "fake"))
        obs = observation()
        for _ in range(4):
            action = await bridge.decide(obs)
            assert action in obs.legal_actions  # random fallback
        assert len(ws.sent) == 2  # after 2 timeouts the AI is not asked anymore
        assert ws.sent[0]["timeout"] == 0.01
        assert bridge.stats()["paused"] is True
    asyncio.run(scenario())
 class OverloadedAIWebSocket:
    """Answers every request immediately with the given error message."""
    def __init__(self, bridge_connection_ref, message):
        self.ref, self.message, self.sent = bridge_connection_ref, message, []
    async def send_json(self, request):
        self.sent.append(request)
        future = self.ref[0].pending[request["request_id"]]
        future.set_result({"type": "error", "request_id": request["request_id"], "message": self.message})
 def test_load_shedding_does_not_trip_the_breaker():
    async def scenario():
        bridge = AIBridge(timeout=1, rng=random.Random(0), breaker_failures=2)
        ref = []
        ws = OverloadedAIWebSocket(ref, "overloaded")
        connection = AIConnection(ws, "fake")
        ref.append(connection)
        bridge._connections.append(connection)
        obs = observation()
        for _ in range(5):
            assert await bridge.decide(obs) in obs.legal_actions
        assert len(ws.sent) == 5  # still asked every time
        assert bridge.stats()["paused"] is False
        ws.message = "model crashed"  # a real failure counts
        for _ in range(3):
            await bridge.decide(obs)
        assert len(ws.sent) == 7 and bridge.stats()["paused"] is True
    asyncio.run(scenario())
 def test_ai_inflight_limit():
    async def scenario():
        bridge = AIBridge(timeout=1, rng=random.Random(0), max_inflight=1)
        ws = SilentWebSocket()
        connection = AIConnection(ws, "fake")
        connection.pending["busy"] = asyncio.get_running_loop().create_future()
        bridge._connections.append(connection)
        obs = observation()
        for _ in range(10):
            assert await bridge.decide(obs) in obs.legal_actions
        assert ws.sent == []  # not even sent: too many requests already waiting
        assert bridge.stats()["paused"] is False  # load shedding is not a failure
    asyncio.run(scenario())
@@ -0,0 +1,137 @@
 """Abuse limits, load-based admission control and server_status messages."""
 import pytest
 from starlette.websockets import WebSocketDisconnect
 from app.load import LoadSample
 from helpers import connect, is_state, receive_until
 def is_error(code):
    return lambda m: m["type"] == "error" and m.get("code") == code
 def test_welcome_is_followed_by_server_status(client):
    ws = client.sockets.enter_context(client.websocket_connect("/ws"))
    ws.send_json({"type": "hello", "token": None})
    assert ws.receive_json()["type"] == "welcome"
    status = ws.receive_json()
    assert status == {"type": "server_status", "load": "normal", "accepting_games": True}
 def test_message_flood_is_limited_and_closed(make_client):
    client = make_client(message_rate=0.001, message_burst=3, max_rate_violations=5)
    ws, _ = connect(client)  # uses 1 message
    for _ in range(2):  # uses the remaining burst
        ws.send_json({"type": "leave_lobby"})
    ws.send_json({"type": "leave_lobby"})  # over the limit
    assert receive_until(ws, is_error("rate_limited"))
    with pytest.raises(WebSocketDisconnect) as closed:
        for _ in range(10):
            ws.send_json({"type": "leave_lobby"})
        for _ in range(20):
            ws.receive_json()
    assert closed.value.code == 1008
 def test_connections_per_ip_are_limited(make_client):
    client = make_client(max_connections_per_ip=2)
    connect(client)
    connect(client)
    ws = client.sockets.enter_context(client.websocket_connect("/ws"))
    assert ws.receive_json()["code"] == "busy"
    with pytest.raises(WebSocketDisconnect) as closed:
        ws.receive_json()
    assert closed.value.code == 1013
 def test_closed_connections_free_their_slot(make_client):
    client = make_client(max_connections_per_ip=1)
    with client.websocket_connect("/ws") as ws:
        ws.send_json({"type": "hello", "token": None})
        receive_until(ws, is_state)
    ws, _ = connect(client)  # works again after the first one closed
 def test_lobby_creation_rate_per_ip(make_client):
    client = make_client(lobby_create_rate=2)
    ws, _ = connect(client)
    for _ in range(2):
        ws.send_json({"type": "create_lobby", "name": "Ann"})
        receive_until(ws, lambda m: is_state(m) and m["lobby"])
    ws.send_json({"type": "create_lobby", "name": "Ann"})
    assert receive_until(ws, is_error("rate_limited"))
 def test_overloaded_server_refuses_new_games(make_client):
    client = make_client(load_force_level="overloaded")
    ws = client.sockets.enter_context(client.websocket_connect("/ws"))
    ws.send_json({"type": "hello", "token": None})
    status = receive_until(ws, lambda m: m["type"] == "server_status")
    assert status["load"] == "overloaded" and not status["accepting_games"]
    ws.send_json({"type": "create_lobby", "name": "Ann"})
    assert receive_until(ws, is_error("busy"))
    assert client.get("/health").json()["accepting_games"] is False
 def start_game_with_ais(client, name):
    ws, _ = connect(client)
    ws.send_json({"type": "create_lobby", "name": name})
    receive_until(ws, lambda m: is_state(m) and m["lobby"])
    ws.send_json({"type": "add_ai"})
    ws.send_json({"type": "add_ai"})
    return ws
 def test_max_active_games(make_client):
    client = make_client(max_active_games=1)
    first = start_game_with_ais(client, "Ann")
    second = start_game_with_ais(client, "Ben")
    first.send_json({"type": "start_game"})
    receive_until(first, lambda m: is_state(m) and m["game"])
    # Everybody learns that no further games can start.
    status = receive_until(second, lambda m: m["type"] == "server_status")
    assert status["accepting_games"] is False
    second.send_json({"type": "start_game"})
    assert receive_until(second, is_error("busy"))
    # Leaving the running game ends it (only AIs left) and frees the slot.
    first.send_json({"type": "leave_lobby"})
    status = receive_until(second, lambda m: m["type"] == "server_status")
    assert status["accepting_games"] is True
    second.send_json({"type": "start_game"})
    assert receive_until(second, lambda m: is_state(m) and m["game"])
 def test_running_game_continues_when_overloaded(client):
    ws = start_game_with_ais(client, "Ann")
    ws.send_json({"type": "start_game"})
    latest = receive_until(ws, lambda m: is_state(m) and m["game"])
    service = client.app.state.service
    for _ in range(3):  # three overloaded samples in a row
        client.portal.call(service.load.record, LoadSample(loop_lag=1.0))
    # normal -> busy (first bad sample) -> overloaded (third in a row).
    # Keep track of the latest game state seen on the way.
    while True:
        message = ws.receive_json()
        if is_state(message):
            latest = message
        if message["type"] == "server_status" and message["load"] == "overloaded":
            assert message["accepting_games"] is False
            break
    # The running game still accepts moves.
    def my_turn(state):
        obs = state["game"]["observation"]
        return obs["current_player"] == state["game"]["you"] and obs["legal_actions"]
    while not my_turn(latest):
        latest = receive_until(ws, is_state)
    ws.send_json({"type": "action", "action": latest["game"]["observation"]["legal_actions"][0]})
    reply = ws.receive_json()
    assert reply["type"] == "state", reply
@@ -1,42 +1,9 @@
 import contextlib
 import random
 import threading
 import pytest
 from fastapi.testclient import TestClient
-from app.config import Settings
+from helpers import SECRET, connect, is_state, receive_until
 from app.main import create_app
 SECRET = "test-secret"
@pytest.fixture
 def client():
    settings = Settings(ai_shared_secret=SECRET, ai_move_delay=0, trick_pause=0, ai_decision_timeout=2)
    with TestClient(create_app(settings)) as test_client, contextlib.ExitStack() as sockets:
        test_client.sockets = sockets  # websocket sessions are closed before the app stops
        yield test_client
 def receive_until(ws, predicate, limit=5000):
    for _ in range(limit):
        message = ws.receive_json()
        if predicate(message):
            return message
    raise AssertionError("expected message not received")
 def is_state(message):
    return message["type"] == "state"
 def connect(client, token=None):
    ws = client.sockets.enter_context(client.websocket_connect("/ws"))
    ws.send_json({"type": "hello", "token": token})
    welcome = receive_until(ws, lambda m: m["type"] == "welcome")
    receive_until(ws, is_state)
    return ws, welcome
 def test_health(client):
@@ -18,9 +18,12 @@ development, or `VITE_WS_URL` at build time to use an absolute websocket URL ins
 ## Structure
 - `src/api/` — protocol types and the `useWizard` hook (websocket connection,
-  reconnect with a per-tab session token, game state)
+  reconnect with a per-tab session token, game state, server load status; slow
  retries when the server is full)
 - `src/components/` — reusable UI (button, playing card, modal, toasts)
- `src/screens/HomeScreen` — enter a name, create or join a lobby
+- `src/screens/HomeScreen` — enter a name, create or join a lobby (Create is disabled with a
- `src/screens/LobbyScreen` — lobby code, players, AI players, rule options, start
+  notice while the server is busy)
 - `src/screens/LobbyScreen` — lobby code, players, AI players, rule options, start (disabled
  with a notice while the server is busy)
 - `src/screens/GameScreen` — the table: seats, trick, hand, bidding, trump choice,
  scoreboard, round summary, game over
@@ -10,10 +10,12 @@ const STATUS_MESSAGES = {
  connecting: 'Connecting to the server…',
  closed: 'Connection lost. Reconnecting…',
  replaced: 'This game was opened in another window. Reload the page to continue here.',
  busy: 'The server is full right now. Retrying in a moment…',
 } as const
 function App() {
-  const { status, playerId, lobby, game, completedTrick, toasts, dismissToast, actions } = useWizard()
+  const { status, playerId, lobby, game, completedTrick, serverStatus, toasts, dismissToast, actions } =
    useWizard()
  const lobbyCode = lobby?.code
  // Keep the lobby code in the address bar so the URL can be shared as an invite link.
@@ -27,9 +29,22 @@ function App() {
  let screen
  if (!lobby || !playerId) {
-    screen = <HomeScreen connected={status === 'open'} actions={actions} />
+    screen = (
      <HomeScreen
        connected={status === 'open'}
        acceptingGames={serverStatus.accepting_games}
        actions={actions}
      />
    )
  } else if (lobby.status === 'waiting' || !game) {
-    screen = <LobbyScreen lobby={lobby} playerId={playerId} actions={actions} />
+    screen = (
      <LobbyScreen
        lobby={lobby}
        playerId={playerId}
        acceptingGames={serverStatus.accepting_games}
        actions={actions}
      />
    )
  } else {
    screen = (
      <GameScreen
@@ -103,10 +103,22 @@ export type GameEvent =
  | { type: 'round_started'; round_number: number; dealer: number; trump_card: Card | null }
  | { type: 'game_finished'; scores: number[]; winners: number[] }
 export type LoadLevel = 'normal' | 'busy' | 'overloaded'
 export interface ServerStatus {
  load: LoadLevel
  // False when the server refuses new lobbies/games (overloaded or full).
  accepting_games: boolean
 }
 // Machine readable error codes: 'busy' (no capacity) or 'rate_limited'.
 export type ErrorCode = 'busy' | 'rate_limited'
 export type ServerMessage =
  | { type: 'welcome'; player_id: string; token: string; name: string | null }
  | { type: 'state'; lobby: Lobby | null; game: GameView | null; events: GameEvent[] }
-  | { type: 'error'; message: string }
+  | ({ type: 'server_status' } & ServerStatus)
  | { type: 'error'; message: string; code?: ErrorCode }
 export type ClientMessage =
  | { type: 'hello'; token: string | null }
@@ -6,10 +6,12 @@ import type {
  GameView,
  Lobby,
  ServerMessage,
  ServerStatus,
  TrickResult,
 } from './types'
-export type ConnectionStatus = 'connecting' | 'open' | 'closed' | 'replaced'
+// 'busy': the server refused the connection because it is full.
 export type ConnectionStatus = 'connecting' | 'open' | 'closed' | 'replaced' | 'busy'
 export interface Toast {
  id: number
@@ -23,6 +25,9 @@ const TOAST_MS = 4000
 const MAX_RETRY_DELAY_MS = 10000
 // Close code the server uses when the session was opened in another tab.
 const CLOSE_REPLACED = 4000
 // "Try Again Later": the server is full. Retry slowly so we don't add load.
 const CLOSE_TRY_AGAIN_LATER = 1013
 const BUSY_RETRY_DELAYS_MS = [15000, 30000, 60000]
 function websocketUrl(): string {
  const configured = import.meta.env.VITE_WS_URL as string | undefined
@@ -59,6 +64,7 @@ export function useWizard() {
  // The most recently completed trick, shown for a moment before it is cleared.
  const [completedTrick, setCompletedTrick] = useState<TrickResult | null>(null)
  const [toasts, setToasts] = useState<Toast[]>([])
  const [serverStatus, setServerStatus] = useState<ServerStatus>({ load: 'normal', accepting_games: true })
  const socketRef = useRef<WebSocket | null>(null)
  const toastId = useRef(0)
@@ -76,6 +82,7 @@ export function useWizard() {
  useEffect(() => {
    let disposed = false
    let attempt = 0
    let busyAttempt = 0
    let retryTimer: ReturnType<typeof setTimeout> | undefined
    let trickTimer: ReturnType<typeof setTimeout> | undefined
@@ -97,6 +104,9 @@ export function useWizard() {
          if (!message.game) setCompletedTrick(null)
          break
        }
        case 'server_status':
          setServerStatus({ load: message.load, accepting_games: message.accepting_games })
          break
        case 'error':
          showError(message.message)
          break
@@ -107,6 +117,9 @@ export function useWizard() {
      const socket = new WebSocket(websocketUrl())
      socketRef.current = socket
      // A "busy" error before the welcome means the server refused this connection.
      let welcomed = false
      let refused = false
      socket.onopen = () => {
        attempt = 0
        setStatus('open')
@@ -114,7 +127,18 @@ export function useWizard() {
      }
      socket.onmessage = (event) => {
        try {
-          handleMessage(JSON.parse(event.data as string) as ServerMessage)
+          const message = JSON.parse(event.data as string) as ServerMessage
          if (message.type === 'welcome') {
            welcomed = true
            busyAttempt = 0
          }
          // The "server is full" error arrives right before the 1013 close; the
          // banner explains it, so no toast.
          if (message.type === 'error' && message.code === 'busy' && !welcomed) {
            refused = true
            return
          }
          handleMessage(message)
        } catch (error) {
          console.error('Invalid message from server', error)
        }
@@ -126,9 +150,17 @@ export function useWizard() {
          setStatus('replaced')
          return
        }
        let delay: number
        if (event.code === CLOSE_TRY_AGAIN_LATER || refused) {
          setStatus('busy')
          delay = BUSY_RETRY_DELAYS_MS[Math.min(busyAttempt, BUSY_RETRY_DELAYS_MS.length - 1)]
          busyAttempt += 1
        } else {
          setStatus('closed')
-        const delay = Math.min(1000 * 2 ** attempt, MAX_RETRY_DELAY_MS)
+          busyAttempt = 0
          delay = Math.min(1000 * 2 ** attempt, MAX_RETRY_DELAY_MS)
          attempt += 1
        }
        retryTimer = setTimeout(() => {
          setStatus('connecting')
          connect()
@@ -173,7 +205,7 @@ export function useWizard() {
    [send],
  )
-  return { status, playerId, lobby, game, completedTrick, toasts, dismissToast, actions }
+  return { status, playerId, lobby, game, completedTrick, serverStatus, toasts, dismissToast, actions }
 }
 export type WizardActions = ReturnType<typeof useWizard>['actions']
@@ -104,3 +104,13 @@
  color: var(--text-muted);
  font-size: 13px;
 }
 .busy {
  margin: 0;
  padding: 10px 12px;
  border: 1px solid rgb(255 209 102 / 40%);
  border-radius: 10px;
  background: rgb(255 209 102 / 8%);
  color: var(--gold);
  font-size: 14px;
 }
@@ -20,10 +20,12 @@ function initialCode(): string {
 interface HomeScreenProps {
  connected: boolean
  // False while the server refuses new games (overloaded or full).
  acceptingGames: boolean
  actions: WizardActions
 }
-export function HomeScreen({ connected, actions }: HomeScreenProps) {
+export function HomeScreen({ connected, acceptingGames, actions }: HomeScreenProps) {
  const [name, setName] = useState(initialName)
  const [code, setCode] = useState(initialCode)
  const trimmedName = name.trim()
@@ -73,9 +75,20 @@ export function HomeScreen({ connected, actions }: HomeScreenProps) {
          onChange={(event) => setName(event.target.value)}
        />
-        <Button size="lg" className={styles.full} disabled={!connected || !trimmedName} onClick={create}>
+        <Button
          size="lg"
          className={styles.full}
          disabled={!connected || !trimmedName || !acceptingGames}
          onClick={create}
        >
          Create lobby
        </Button>
        {connected && !acceptingGames && (
          <p className={styles.busy} role="status">
            The server is busy right now, so new games can't be created. You can still join an existing
            lobby — please try again in a moment.
          </p>
        )}
        <div className={styles.divider}>
          <span>or join a lobby</span>
@@ -211,3 +211,13 @@
  color: var(--text-muted);
  font-size: 14px;
 }
 .busy {
  margin: 0;
  padding: 12px 16px;
  border: 1px solid rgb(255 209 102 / 40%);
  border-radius: 12px;
  background: rgb(255 209 102 / 8%);
  color: var(--gold);
  font-size: 14px;
 }
@@ -7,6 +7,8 @@ import styles from './LobbyScreen.module.css'
 interface LobbyScreenProps {
  lobby: Lobby
  playerId: string
  // False while the server refuses new games (overloaded or full).
  acceptingGames: boolean
  actions: WizardActions
 }
@@ -17,7 +19,7 @@ function inviteLink(code: string): string {
  return url.toString()
 }
-export function LobbyScreen({ lobby, playerId, actions }: LobbyScreenProps) {
+export function LobbyScreen({ lobby, playerId, acceptingGames, actions }: LobbyScreenProps) {
  const [copied, setCopied] = useState<'code' | 'link' | null>(null)
  const isHost = lobby.host_id === playerId
  const isFull = lobby.players.length >= lobby.max_players
@@ -116,13 +118,24 @@ export function LobbyScreen({ lobby, playerId, actions }: LobbyScreenProps) {
        </label>
      </section>
      {!acceptingGames && (
        <p className={styles.busy} role="status">
          The server is busy right now, so games can't be started. Your lobby stays open — try again in a
          moment.
        </p>
      )}
      <footer className={styles.actions}>
        <Button variant="danger" onClick={actions.leaveLobby}>
          Leave lobby
        </Button>
        {isHost ? (
-          <Button size="lg" disabled={missing > 0} onClick={actions.startGame}>
+          <Button size="lg" disabled={missing > 0 || !acceptingGames} onClick={actions.startGame}>
-            {missing > 0 ? `Need ${missing} more player${missing > 1 ? 's' : ''}` : 'Start game'}
+            {missing > 0
              ? `Need ${missing} more player${missing > 1 ? 's' : ''}`
              : acceptingGames
                ? 'Start game'
                : 'Server busy'}
          </Button>
        ) : (
          <p className={styles.waiting}>Waiting for the host to start the game…</p>