removed cloud storage

The model returned cells as [row, column] while the dataset targets,
convert_prediction and the metrics use [column, row], so most cells
were trained on faces outside the region they can see.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.gitignore

@@ -1,6 +1,7 @@
 api/data/
 api/data.*
 api/saved_models/
+api/runs/
 api/testimages/
 **/__pycache__/
 

api/src/bird_cnn/config.py

@@ -28,6 +28,10 @@ class Config:
     model_name: str = "bird_cnn"
     save: bool = True
 
+    # Run log (src/common/run_log.py): one folder per training run
+    runs_dir: str = "runs"            # empty = no run log
+    run_name: str = ""                # added to the run folder name, e.g. "bn-lr3e-4"
+
     # predict task
     image: str = "test1.jpg"
     checkpoint: str = ""                          # empty = <save_dir>/<model_name>/<model_name>

api/src/bird_cnn/tasks.py

@@ -56,7 +56,7 @@ def train(cfg: Config):
 
     fit(model, nn.CrossEntropyLoss(), optimizer, train_loader, val_loader, device,
         epochs=cfg.epochs, save_dir=cfg.save_dir, model_name=cfg.model_name, save=cfg.save,
-        monitor="acc", track_accuracy=True)
+        monitor="acc", track_accuracy=True, config=cfg)
 
 
 def view_data(cfg: Config):

api/src/common/boxes.py

@@ -54,10 +54,13 @@ def image_to_cell_coords(x, y, img_w, img_h, S, cell_i, cell_j):
     return x - (img_w / S) * cell_i, y - (img_h / S) * cell_j
 
 
+def grid_cell(x, y, img_w, img_h, S):
+    """Image position -> (cell_i, cell_j) of the S x S grid cell it falls in."""
+    return min(int(x / (img_w / S)), S - 1), min(int(y / (img_h / S)), S - 1)
+
+
 def is_center_in_grid_cell(x, y, img_w, img_h, S, cell_i, cell_j):
-    gt_cell_i = min(int(x / (img_w / S)), S - 1)
+    return grid_cell(x, y, img_w, img_h, S) == (cell_i, cell_j)
-    gt_cell_j = min(int(y / (img_h / S)), S - 1)
-    return (gt_cell_i == cell_i) and (gt_cell_j == cell_j)
 
 
 def scale_box(box, scale_w, scale_h):

api/src/common/checkpoints.py

@@ -3,13 +3,16 @@ from pathlib import Path
 import torch
 
 
-def checkpoint_path(save_dir: str | Path, model_name: str) -> Path:
+def checkpoint_path(save_dir: str | Path, model_name: str, epoch: int | None = None) -> Path:
-    # Layout kept from the original code: <save_dir>/<name>/<name>
+    # Layout kept from the original code: <save_dir>/<name>/<name>,
-    return Path(save_dir) / model_name / model_name
+    # per-epoch checkpoints next to it as <name>_epoch007
+    file_name = model_name if epoch is None else f"{model_name}_epoch{epoch:03d}"
+    return Path(save_dir) / model_name / file_name
 
 
-def save_checkpoint(model: torch.nn.Module, save_dir: str | Path, model_name: str) -> Path:
+def save_checkpoint(model: torch.nn.Module, save_dir: str | Path, model_name: str,
-    path = checkpoint_path(save_dir, model_name)
+                    epoch: int | None = None) -> Path:
+    path = checkpoint_path(save_dir, model_name, epoch)
     path.parent.mkdir(parents=True, exist_ok=True)
     torch.save(model.state_dict(), path)
     return path

api/src/common/run_log.py

@@ -0,0 +1,197 @@
+"""Records every training run in its own folder, so runs with different
+settings or architectures can be compared later.
+
+    <runs_dir>/<model_name>/<YYYYmmdd-HHMMSS>[_<run_name>]/
+        run.json          everything about the run: config, command, model
+                          (class, parameter counts, architecture hash),
+                          optimizer, loss, data sizes, environment, git state
+                          and the result (status, best epoch and metric,
+                          duration). Rewritten after every epoch.
+        metrics.csv       one row per epoch
+        architecture.txt  the model's module tree, str(model)
+        changes.diff      uncommitted code changes (only if there are any)
+        best.pt           state_dict of the best epoch so far
+
+result.status in run.json is "running" during training, then "completed",
+"failed" or "interrupted". A run whose process was killed stays "running".
+"""
+
+import csv
+import dataclasses
+import hashlib
+import json
+import os
+import platform
+import re
+import socket
+import subprocess
+import sys
+import time
+from datetime import datetime
+from pathlib import Path
+
+import torch
+
+REPO_DIR = Path(__file__).resolve().parent
+
+
+def _git(*args):
+    try:
+        return subprocess.run(["git", *args], cwd=REPO_DIR, capture_output=True, text=True,
+                              timeout=10, check=True).stdout
+    except (OSError, subprocess.SubprocessError):
+        return None
+
+
+def _jsonable(value):
+    if value is None or isinstance(value, (str, int, float, bool)):
+        return value
+    if isinstance(value, (list, tuple)):
+        return [_jsonable(v) for v in value]
+    if isinstance(value, dict):
+        return {str(k): _jsonable(v) for k, v in value.items()}
+    if isinstance(value, torch.Tensor):
+        return value.tolist() if value.numel() <= 16 else f"tensor{tuple(value.shape)}"
+    return str(value)
+
+
+def _module_settings(module):
+    """Public attributes and small buffers, e.g. a loss's reduction and class weights."""
+    settings = {k: v for k, v in vars(module).items() if not k.startswith("_") and k != "training"}
+    settings.update(dict(module.named_buffers()))
+    return _jsonable(settings)
+
+
+def _dataset_size(loader):
+    try:
+        return len(loader.dataset)
+    except TypeError:
+        return None
+
+
+class RunLog:
+    def __init__(self, runs_dir, model_name, config, model, optimizer, loss_fn,
+                 train_loader, val_loader, device, monitor):
+        self.monitor = monitor
+        self.started = time.time()
+        self.dir = self._make_dir(Path(runs_dir) / model_name, getattr(config, "run_name", ""))
+        self._csv_fields = None
+
+        architecture = str(model)
+        status = _git("status", "--porcelain")
+        diff = _git("diff", "HEAD")
+        gpu = torch.cuda.get_device_name(device) if torch.device(device).type == "cuda" else None
+
+        self.info = {
+            "id": f"{model_name}/{self.dir.name}",
+            "model_name": model_name,
+            "started": datetime.fromtimestamp(self.started).isoformat(timespec="seconds"),
+            "command": " ".join(sys.argv),
+            "config": _jsonable(dataclasses.asdict(config)),
+            "model": {
+                "class": type(model).__name__,
+                "params": sum(p.numel() for p in model.parameters()),
+                "trainable_params": sum(p.numel() for p in model.parameters() if p.requires_grad),
+                "architecture_hash": hashlib.sha256(architecture.encode()).hexdigest()[:12],
+            },
+            "optimizer": {
+                "class": type(optimizer).__name__,
+                **_jsonable({k: v for k, v in optimizer.param_groups[0].items() if k != "params"}),
+            },
+            "loss": {"class": type(loss_fn).__name__, **_module_settings(loss_fn)},
+            "data": {
+                "train_size": _dataset_size(train_loader),
+                "val_size": _dataset_size(val_loader),
+                "batch_size": train_loader.batch_size,
+                "train_workers": train_loader.num_workers,
+            },
+            "environment": {
+                "host": socket.gethostname(),
+                "platform": platform.platform(),
+                "python": platform.python_version(),
+                "torch": torch.__version__,
+                "cuda": torch.version.cuda,
+                "device": str(device),
+                "gpu": gpu,
+                "seed": torch.initial_seed(),
+            },
+            "git": {
+                "commit": (_git("rev-parse", "HEAD") or "").strip() or None,
+                "branch": (_git("rev-parse", "--abbrev-ref", "HEAD") or "").strip() or None,
+                "dirty": bool(status.strip()) if status is not None else None,
+                "changed_files": [line[3:] for line in status.splitlines()] if status else [],
+            },
+            "result": {
+                "status": "running",
+                "monitor": monitor,
+                "epochs_done": 0,
+                "best_epoch": None,
+                f"best_val_{monitor}": None,
+                "last": None,
+                "duration_s": 0,
+                "finished": None,
+                "error": None,
+            },
+        }
+
+        (self.dir / "architecture.txt").write_text(architecture + "\n")
+        if diff:
+            (self.dir / "changes.diff").write_text(diff)
+        self._write_info()
+        print(f"Logging run to {self.dir}")
+
+    @staticmethod
+    def _make_dir(parent, run_name):
+        name = datetime.now().strftime("%Y%m%d-%H%M%S")
+        if run_name:
+            name += "_" + re.sub(r"[^\w.-]+", "-", run_name)
+        parent.mkdir(parents=True, exist_ok=True)
+        for i in range(1, 100):
+            path = parent / (name if i == 1 else f"{name}-{i}")
+            try:
+                path.mkdir()
+                return path
+            except FileExistsError:
+                continue
+        raise FileExistsError(f"no free run folder for {parent / name}")
+
+    def _write_info(self):
+        # Write-then-rename so a reader never sees a half written file.
+        tmp = self.dir / "run.json.tmp"
+        tmp.write_text(json.dumps(self.info, indent=2) + "\n")
+        os.replace(tmp, self.dir / "run.json")
+
+    def log_epoch(self, epoch, train, val, lr, seconds, improved):
+        row = {"epoch": epoch,
+               **{f"train_{k}": v for k, v in train.items()},
+               **{f"val_{k}": v for k, v in val.items()},
+               "lr": lr, "seconds": round(seconds, 2), "improved": improved}
+        path = self.dir / "metrics.csv"
+        with path.open("a", newline="") as f:
+            if self._csv_fields is None:
+                self._csv_fields = list(row)
+                writer = csv.DictWriter(f, fieldnames=self._csv_fields)
+                writer.writeheader()
+            else:
+                writer = csv.DictWriter(f, fieldnames=self._csv_fields)
+            writer.writerow(row)
+
+        result = self.info["result"]
+        result["epochs_done"] = epoch
+        result["last"] = {k: v for k, v in row.items() if k != "improved"}
+        if improved:
+            result["best_epoch"] = epoch
+            result[f"best_val_{self.monitor}"] = val[self.monitor]
+        result["duration_s"] = round(time.time() - self.started, 1)
+        self._write_info()
+
+    def save_weights(self, model):
+        torch.save(model.state_dict(), self.dir / "best.pt")
+
+    def finish(self, status, error=None):
+        result = self.info["result"]
+        result["status"] = status
+        result["error"] = error
+        result["duration_s"] = round(time.time() - self.started, 1)
+        result["finished"] = datetime.now().isoformat(timespec="seconds")
+        self._write_info()

api/src/common/training.py

@@ -2,7 +2,13 @@
 
 Each epoch trains, validates, prints the metrics and saves the model when the
 monitored metric improves. `prepare_batch` turns a loader batch into
-(inputs, targets); the default just moves both to the device."""
+(inputs, targets); the default just moves both to the device.
+
+An optional `scheduler` is stepped once after every epoch, so the learning
+rate logged for an epoch is the one it trained with.
+
+Passing the task's `config` also records the run (settings, architecture,
+metrics per epoch, best weights) under config.runs_dir, see run_log.py."""
 
 import math
 import time
@@ -12,6 +18,7 @@ from torch.utils.data import DataLoader
 from tqdm import tqdm
 
 from src.common.checkpoints import save_checkpoint
+from src.common.run_log import RunLog
 
 
 def _default_prepare(batch, device):
@@ -19,34 +26,72 @@ def _default_prepare(batch, device):
     return inputs.to(device), targets.to(device)
 
 
-def _run_epoch(model, loss_fn, loader, device, prepare_batch, track_accuracy, optimizer=None, desc=""):
+def adamw(model, lr, weight_decay):
+    """AdamW that decays only conv/linear weights. Biases and BatchNorm
+    parameters (everything 1-D) are left alone, shrinking them only hurts."""
+    decay = [p for p in model.parameters() if p.requires_grad and p.ndim > 1]
+    no_decay = [p for p in model.parameters() if p.requires_grad and p.ndim <= 1]
+    return torch.optim.AdamW([{"params": decay, "weight_decay": weight_decay},
+                              {"params": no_decay, "weight_decay": 0.0}], lr=lr)
+
+
+def warmup_cosine(optimizer, epochs, warmup_epochs, min_lr):
+    """Per-epoch schedule: linear warmup from lr/10 over warmup_epochs, then
+    cosine decay from lr to min_lr over the remaining epochs."""
+    floor = min_lr / optimizer.param_groups[0]["lr"]
+    decay_epochs = max(epochs - warmup_epochs - 1, 1)
+
+    def factor(epoch):  # epoch counts from 0
+        if epoch < warmup_epochs:
+            return 0.1 + 0.9 * epoch / warmup_epochs
+        progress = min(epoch - warmup_epochs, decay_epochs) / decay_epochs
+        return floor + (1 - floor) * 0.5 * (1 + math.cos(math.pi * progress))
+
+    return torch.optim.lr_scheduler.LambdaLR(optimizer, factor)
+
+
+def _run_epoch(model, loss_fn, loader, device, prepare_batch, track_accuracy, optimizer=None,
+               val_metrics=None, desc=""):
     training = optimizer is not None
     model.train(training)
 
     loss_sum, correct, count = 0.0, 0, 0
+    all_preds, all_targets = [], []
     with torch.set_grad_enabled(training):
         for batch in tqdm(loader, desc=desc, leave=False):
             inputs, targets = prepare_batch(batch, device)
 
             preds = model(inputs)
-            loss = loss_fn(preds, targets)
 
             if training:
+                loss = loss_fn(preds, targets)
                 optimizer.zero_grad()
                 loss.backward()
                 optimizer.step()
+                loss_sum += loss.item() * inputs.size(0)
+            else:
+                all_preds.append(preds)
+                all_targets.append(targets)
 
-            loss_sum += loss.item()
             count += inputs.size(0)
             if track_accuracy:
                 correct += (preds.argmax(dim=1) == targets).sum().item()
 
-    if torch.cuda.is_available():
+    if training:
-        torch.cuda.empty_cache()
+        metrics = {"loss": loss_sum / max(count, 1)}
-
+    else:
-    metrics = {"loss": loss_sum / max(count, 1)}
+        # Validation loss over the whole set at once, so it doesn't depend on
+        # the batch size (YoloLoss averages per batch over the cells with a face).
+        preds, targets = torch.cat(all_preds), torch.cat(all_targets)
+        metrics = {"loss": loss_fn(preds, targets).item()}
+        if val_metrics:
+            metrics.update(val_metrics(preds, targets))
+        del preds, targets, all_preds, all_targets
     if track_accuracy:
         metrics["acc"] = correct / max(count, 1)
+
+    if torch.cuda.is_available():
+        torch.cuda.empty_cache()
     return metrics
 
 
@@ -61,31 +106,71 @@ def fit(
     save_dir: str,
     model_name: str,
     save: bool = True,
-    monitor: str = "loss",          # "loss" (lower is better) or "acc" (higher is better)
+    monitor: str = "loss",          # "loss" (lower is better) or any other val metric (higher is better)
     track_accuracy: bool = False,   # for classifiers
     prepare_batch=_default_prepare,
+    val_metrics=None,               # f(preds, targets) -> dict of extra metrics over the whole val set
+    config=None,                    # the task's Config; enables the run log
+    scheduler=None,                 # LR scheduler, stepped once per epoch
+    save_every_epoch: bool = False, # with save: also keep every epoch as <model_name>_epochNNN
 ):
+    run = None
+    if config is not None and config.runs_dir:
+        run = RunLog(config.runs_dir, model_name, config, model, optimizer, loss_fn,
+                     train_loader, val_loader, device, monitor)
+    try:
+        best = _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epochs,
+                           save_dir, model_name, save, monitor, track_accuracy, prepare_batch, val_metrics, run,
+                           scheduler, save_every_epoch)
+    except KeyboardInterrupt:
+        if run:
+            run.finish("interrupted")
+        raise
+    except Exception as e:
+        if run:
+            run.finish("failed", error=repr(e))
+        raise
+    if run:
+        run.finish("completed")
+    return best
+
+
+def _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epochs,
+                save_dir, model_name, save, monitor, track_accuracy, prepare_batch, val_metrics, run,
+                scheduler, save_every_epoch):
     best = math.inf if monitor == "loss" else -math.inf
 
     for epoch in range(1, epochs + 1):
+        start = time.perf_counter()
         train = _run_epoch(model, loss_fn, train_loader, device, prepare_batch, track_accuracy,
                            optimizer=optimizer, desc=f"Train {epoch}")
         val = _run_epoch(model, loss_fn, val_loader, device, prepare_batch, track_accuracy,
-                         desc=f"Validate {epoch}")
+                         val_metrics=val_metrics, desc=f"Validate {epoch}")
 
         improved = val[monitor] < best if monitor == "loss" else val[monitor] > best
         if improved:
             best = val[monitor]
             if save:
                 save_checkpoint(model, save_dir, model_name)
+            if run:
+                run.save_weights(model)
+        if save and save_every_epoch:
+            save_checkpoint(model, save_dir, model_name, epoch=epoch)
+        if run:
+            run.log_epoch(epoch, train, val, lr=optimizer.param_groups[0]["lr"],
+                          seconds=time.perf_counter() - start, improved=improved)
+        if scheduler:
+            scheduler.step()
 
-        # Loss is the summed batch loss per sample, shown x1000.
         parts = [f"epoch: {epoch}"]
         if track_accuracy:
             parts.append(f"train acc: {train['acc'] * 100:.1f}% | val acc: {val['acc'] * 100:.1f}%")
-        parts.append(f"train loss: {train['loss'] * 1000:.2f} | val loss: {val['loss'] * 1000:.2f}")
+        parts.append(f"train loss: {train['loss']:.4f} | val loss: {val['loss']:.4f}")
+        parts += [f"val {k}: {v:.3f}" for k, v in val.items() if k not in ("loss", "acc")]
         if improved and save:
             parts.append("saved")
+        if save and save_every_epoch:
+            parts.append(f"kept epoch {epoch}")
         print(" | ".join(parts))
 
     return best

api/src/models/yolo.py

@@ -14,32 +14,30 @@ class SkipBlock(nn.Module):
         super().__init__()
         self.conv = nn.Sequential(
             nn.Conv2d(c_in, c_out, kernel_size, padding=kernel_size // 2),
-            nn.GroupNorm(num_groups=c_out // 8, num_channels=c_out),
+            nn.BatchNorm2d(c_out),
             nn.LeakyReLU(inplace=True),
 
             nn.Conv2d(c_out, c_out, kernel_size, padding=kernel_size // 2),
-            nn.GroupNorm(num_groups=c_out // 8, num_channels=c_out),
+            nn.BatchNorm2d(c_out),
             nn.LeakyReLU(inplace=True),
 
             nn.Conv2d(c_out, c_out, kernel_size, padding=kernel_size // 2),
-            nn.GroupNorm(num_groups=c_out // 8, num_channels=c_out),
+            nn.BatchNorm2d(c_out),
             nn.LeakyReLU(inplace=True)
         )
         self.conv_skip = nn.Sequential(
             nn.Conv2d(c_in, c_out, 1),
-            nn.GroupNorm(num_groups=c_out // 8, num_channels=c_out),
+            nn.BatchNorm2d(c_out),
             nn.LeakyReLU(inplace=True)
         )
 
     def forward(self, x):
-        # F.dropout defaults to training=True, so this stays active in eval
+        return F.leaky_relu(self.conv_skip(x) + self.conv(x), inplace=True)
-        # mode too. Kept as-is because the saved models were trained this way.
-        return F.dropout(F.leaky_relu(self.conv_skip(x) + self.conv(x), inplace=True), p=0.3)
 
 
 class Yolo_model(nn.Module):
-    """Output: [batch, grid, grid, boxes*5 + labels] with per cell
+    """Output: [batch, grid x (column), grid y (row), boxes*5 + labels] with
-    (x, y, w, h, confidence, classes...).
+    per cell (x, y, w, h, confidence, classes...).
 
     size_activation decides how w/h are produced:
       "exp"     - exp(x), used by the deployed build_models/face_detection_yolo
@@ -53,8 +51,8 @@ class Yolo_model(nn.Module):
         self.size_activation = size_activation
 
         self.model = nn.Sequential(
-            nn.Conv2d(c_in, c_hidden, kernel_size=3, padding=1),
+            nn.Conv2d(c_in, c_hidden, kernel_size=3, padding=1, stride=2),
-            nn.GroupNorm(num_groups=c_hidden // 8, num_channels=c_hidden),
+            nn.BatchNorm2d(c_hidden),
             nn.LeakyReLU(inplace=True),
 
             SkipBlock(c_in=c_hidden, c_out=c_hidden),
@@ -62,27 +60,35 @@ class Yolo_model(nn.Module):
             SkipBlock(c_in=c_hidden, c_out=c_hidden),
             SkipBlock(c_in=c_hidden, c_out=c_hidden),
 
-            SkipBlock(c_in=c_hidden, c_out=c_hidden * 2),
+            nn.Conv2d(c_hidden, c_hidden * 2, kernel_size=5, padding=2, stride=4),
+            nn.LeakyReLU(inplace=True),
+
+            SkipBlock(c_in=c_hidden * 2, c_out=c_hidden * 2),
             SkipBlock(c_in=c_hidden * 2, c_out=c_hidden * 2),
             SkipBlock(c_in=c_hidden * 2, c_out=c_hidden * 2),
             SkipBlock(c_in=c_hidden * 2, c_out=c_hidden * 2),
+
+            nn.Conv2d(c_hidden * 2, c_hidden * 4, kernel_size=3, padding=1, stride=2),
+            nn.LeakyReLU(inplace=True),
 
-            SkipBlock(c_in=c_hidden * 2, c_out=c_hidden * 4),
             SkipBlock(c_in=c_hidden * 4, c_out=c_hidden * 4),
             SkipBlock(c_in=c_hidden * 4, c_out=c_hidden * 4),
             SkipBlock(c_in=c_hidden * 4, c_out=c_hidden * 4),
+            SkipBlock(c_in=c_hidden * 4, c_out=c_hidden * 4),
+
+            nn.Conv2d(c_hidden * 4, c_hidden * 8, kernel_size=3, padding=1, stride=2),
+            nn.LeakyReLU(inplace=True),
 
-            nn.Conv2d(c_hidden * 4, c_hidden * 8, kernel_size=3, padding=1),
+            nn.Conv2d(c_hidden * 8, c_hidden * 8, kernel_size=3, padding=1),
-            nn.GroupNorm(num_groups=c_hidden // 2, num_channels=c_hidden * 8),
+            nn.BatchNorm2d(c_hidden * 8),
             nn.LeakyReLU(inplace=True),
-            nn.Dropout(0.3),
 
-            nn.AdaptiveAvgPool2d((grid, grid)),
             nn.Conv2d(c_hidden * 8, boxes * 5 + labels, kernel_size=1)
         )
 
     def forward(self, x):
-        x = self.model(x).permute(0, 2, 3, 1)
+        # [batch, C, rows, cols] -> [batch, cols, rows, C]: cell [x, y] like the targets
+        x = self.model(x).permute(0, 3, 2, 1)
         if self.size_activation == "sigmoid":
             return F.sigmoid(x)
         center = F.sigmoid(x[..., :2])

api/src/production/settings.py

@@ -24,12 +24,12 @@ BIRD_MODEL_FILE = "bird_cnn"
 BIRD_IMAGE_SIZE = 64
 BIRD_C_HIDDEN = 16
 
-FACE_MODEL_FILE = "face_detection_yolo"
+FACE_MODEL_FILE = "face_detection_yolo_3"
-FACE_IMAGE_SIZE = 64
+FACE_IMAGE_SIZE = 256
-FACE_GRID = 6
+FACE_GRID = 8
 FACE_C_HIDDEN = 16
-FACE_SIZE_ACTIVATION = "exp"   # must match how the deployed weights were trained
+FACE_SIZE_ACTIVATION = "sigmoid"   # must match how the deployed weights were trained
-FACE_THRESHOLD = 0.9
+FACE_THRESHOLD = 0.5
 
 # ---- Resource limits (the server is weak, keep everything small and bounded) ----
 # Number of predictions (bird or face) that may run at the same time. (env)

api/src/r_cnn/config.py

@@ -40,6 +40,10 @@ class Config:
     football_model_name: str = "object_detection"
     save: bool = True
 
+    # Run log (src/common/run_log.py): one folder per training run
+    runs_dir: str = "runs"            # empty = no run log
+    run_name: str = ""                # added to the run folder name, e.g. "bn-lr3e-4"
+
     # detect tasks: proposals smaller than min_size or larger than max_size are ignored
     image: str = ""                       # detect-faces: image to run on
     min_size: int = 5

api/src/r_cnn/tasks.py

@@ -71,7 +71,7 @@ def train_faces(cfg: Config):
 
     fit(model, nn.CrossEntropyLoss(weight=weights), optimizer, train_loader, val_loader, device,
         epochs=cfg.epochs, save_dir=cfg.save_dir, model_name=cfg.faces_model_name, save=cfg.save,
-        monitor="loss", track_accuracy=True)
+        monitor="loss", track_accuracy=True, config=cfg)
 
 
 def detect_faces(cfg: Config):
@@ -103,7 +103,7 @@ def train_football(cfg: Config):
 
     fit(model, nn.CrossEntropyLoss(), optimizer, train_loader, val_loader, device,
         epochs=cfg.epochs, save_dir=cfg.save_dir, model_name=cfg.football_model_name, save=cfg.save,
-        monitor="loss", track_accuracy=True, prepare_batch=prepare_batch)
+        monitor="loss", track_accuracy=True, prepare_batch=prepare_batch, config=cfg)
 
 
 def detect_football(cfg: Config):

api/src/yolo/config.py

@@ -15,6 +15,15 @@ class Config:
     image_size: int = 64
     grid: int = 6
 
+    # convert-wider: WIDER FACE -> COCO folders <wider_out_dir>/train and /test
+    wider_dir: str = ""               # contains WIDER_train/, WIDER_val/, wider_face_split/
+    wider_out_dir: str = "data/wider_face"
+    wider_min_face: float = 2.0       # drop smaller faces (px after resizing to image_size)
+
+    # merge-coco: combine COCO folders into one
+    merge_dirs: str = ""              # comma separated, e.g. data/wider_faces_coco/train,data/faces_scenery/train
+    merge_out_dir: str = ""
+
     # Model
     c_hidden: int = 16
     boxes: int = 1
@@ -24,8 +33,10 @@ class Config:
     # Training
     epochs: int = 500
     batch_size: int = 64
-    learning_rate: float = 1e-3
+    learning_rate: float = 1e-3       # peak AdamW learning rate
-    weight_decay: float = 1e-4
+    weight_decay: float = 0.05        # AdamW, conv weights only (not biases / BatchNorm)
+    warmup_epochs: int = 3            # linear warmup from learning_rate / 10
+    min_learning_rate: float = 1e-5   # cosine decay ends here at the last epoch
     train_workers: int = 2
     val_workers: int = 0
     device: str = "auto"              # "auto", "cpu", "cuda:0", ...
@@ -34,6 +45,11 @@ class Config:
     save_dir: str = "saved_models"
     model_name: str = "face_detection_yolo"
     save: bool = True
+    save_every_epoch: bool = False    # with --save: also keep every epoch as <model_name>_epochNNN
+
+    # Run log (src/common/run_log.py): one folder per training run
+    runs_dir: str = "runs"            # empty = no run log
+    run_name: str = ""                # added to the run folder name, e.g. "bn-lr3e-4"
 
     # sample / webcam tasks
     checkpoint: str = ""              # empty = <save_dir>/<model_name>/<model_name>

api/src/yolo/loss.py

@@ -13,40 +13,39 @@ class YoloLoss(nn.Module):
         target_boxes = targets[..., :4]
         target_conf = targets[..., 4]
         target_classes = targets[..., 5:]
+        N = predictions.shape[0]
 
         obj_mask = targets[..., 4] == 1
         noobj_mask = targets[..., 4] == 0
 
 
-        if obj_mask.any():
+        box_loss = F.mse_loss(
-            box_loss = F.mse_loss(
+            pred_boxes[obj_mask],
-                pred_boxes[obj_mask],
+            target_boxes[obj_mask],
-                target_boxes[obj_mask],
+            reduction="sum"
-                reduction="mean"
+        ) if obj_mask.any() else torch.tensor(0.0, device=predictions.device)
-            )
-        else:
-            box_loss = torch.tensor(0.0, device=predictions.device)
 
 
         obj_loss = F.mse_loss(
             pred_conf[obj_mask],
             target_conf[obj_mask],
-            reduction="mean"
+            reduction="sum"
         ) if obj_mask.any() else torch.tensor(0.0, device=predictions.device)
 
         noobj_loss = F.mse_loss(
             pred_conf[noobj_mask],
             target_conf[noobj_mask],
-            reduction="mean"
+            reduction="sum"
         ) if noobj_mask.any() else torch.tensor(0.0, device=predictions.device)
 
 
-        class_loss = F.binary_cross_entropy_with_logits(
+        class_loss = F.binary_cross_entropy(
             pred_classes[obj_mask],
-            target_classes[obj_mask]
+            target_classes[obj_mask],
+            reduction="sum"
         ) if obj_mask.any() else torch.tensor(0.0, device=predictions.device)
 
         
-        total_loss = box_loss + obj_loss + noobj_loss + class_loss
+        total_loss = (box_loss * 5 + obj_loss + noobj_loss * 0.5 + class_loss) / N
 
         return total_loss

api/src/yolo/merge.py

@@ -0,0 +1,94 @@
+"""Combine COCO folders (images + <annotation_file>) into one COCO folder
+that YoloDataset reads, e.g. WIDER and faces_scenery into one training set.
+
+- Images are hard-linked into the output folder (copied if the folders are
+  on different drives), named "<source>__<file_name>" so names never clash.
+- Image and annotation ids are renumbered.
+- Categories are merged by name and numbered 1, 2, ... in order of first
+  appearance, so the same class in both sources becomes one class. --labels
+  must equal the number of merged categories.
+"""
+
+import json
+import os
+import re
+import shutil
+from pathlib import Path
+
+from src.yolo.config import Config
+
+
+def source_names(folders):
+    """Readable, unique prefix per source: data/wider_faces_coco/train -> wider_faces_coco_train."""
+    names = []
+    for folder in folders:
+        name = re.sub(r"[^\w.-]+", "_", "_".join(folder.parts[-2:]))
+        while name in names:
+            name += "_"
+        names.append(name)
+    return names
+
+
+def link_or_copy(source: Path, target: Path):
+    try:
+        os.link(source, target)
+    except OSError:
+        shutil.copy2(source, target)
+
+
+def merge_coco(cfg: Config):
+    """Combine the COCO folders in --merge-dirs (comma separated) into --merge-out-dir."""
+    folders = [Path(d.strip()) for d in cfg.merge_dirs.split(",") if d.strip()]
+    if len(folders) < 2:
+        raise SystemExit("Set --merge-dirs to at least two COCO folders, e.g. data/a/train,data/b/train")
+    if not cfg.merge_out_dir:
+        raise SystemExit("Set --merge-out-dir to the folder the combined dataset is written to")
+    out = Path(cfg.merge_out_dir)
+    if out.exists():
+        raise SystemExit(f"{out} already exists, delete it or pick another --merge-out-dir")
+
+    # read and check everything before writing anything
+    sources = []
+    for folder in folders:
+        annotation_path = folder / cfg.annotation_file
+        if not annotation_path.is_file():
+            raise SystemExit(f"{annotation_path} not found")
+        coco = json.loads(annotation_path.read_text())
+        missing = [i["file_name"] for i in coco["images"] if not (folder / i["file_name"]).is_file()]
+        if missing:
+            raise SystemExit(f"{folder}: {len(missing)} images listed in {cfg.annotation_file} "
+                             f"are missing, e.g. {missing[0]}")
+        sources.append(coco)
+
+    categories = {}  # name -> merged category
+    images, annotations = [], []
+    out.mkdir(parents=True)
+
+    for folder, name, coco in zip(folders, source_names(folders), sources):
+        category_ids = {}
+        for category in coco["categories"]:
+            merged = categories.setdefault(category["name"], {**category, "id": len(categories) + 1})
+            category_ids[category["id"]] = merged["id"]
+
+        image_ids = {}
+        for image in coco["images"]:
+            # flat output folder: keep subfolders of the source in the name
+            file_name = f"{name}__{Path(image['file_name']).as_posix().replace('/', '__')}"
+            link_or_copy(folder / image["file_name"], out / file_name)
+            image_ids[image["id"]] = len(images) + 1
+            images.append({**image, "id": len(images) + 1, "file_name": file_name})
+
+        for annotation in coco["annotations"]:
+            annotations.append({**annotation, "id": len(annotations) + 1,
+                                "image_id": image_ids[annotation["image_id"]],
+                                "category_id": category_ids[annotation["category_id"]]})
+
+        print(f"{folder}: {len(coco['images'])} images, {len(coco['annotations'])} annotations "
+              f"(prefix {name}__)")
+
+    merged = {"images": images, "annotations": annotations, "categories": list(categories.values())}
+    (out / cfg.annotation_file).write_text(json.dumps(merged))
+
+    names = ", ".join(f"{c['id']} = {c['name']}" for c in merged["categories"])
+    print(f"{out}: {len(images)} images, {len(annotations)} annotations, categories: {names}")
+    print(f"Train with --labels {len(categories)}")

api/src/yolo/metrics.py

@@ -0,0 +1,51 @@
+"""Detection metrics over the whole validation set, logged next to the loss.
+Per grid cell, like the loss: a cell is a detection when its confidence
+passes `conf_threshold`, and a correct one when the cell holds a face and
+the box overlaps the true box by at least `iou_threshold`."""
+
+import torch
+
+
+def _cell_boxes(t):
+    """[N, S, S, >=4] cell outputs (x, y relative to the cell, w, h relative to
+    the image) -> [N, S, S, 4] boxes as (xmin, ymin, xmax, ymax) in image units."""
+    S = t.shape[1]
+    i = torch.arange(S, device=t.device).view(S, 1)
+    j = torch.arange(S, device=t.device).view(1, S)
+    cx = (i + t[..., 0]) / S
+    cy = (j + t[..., 1]) / S
+    w, h = t[..., 2], t[..., 3]
+    return torch.stack([cx - w / 2, cy - h / 2, cx + w / 2, cy + h / 2], dim=-1)
+
+
+def _iou(a, b):
+    lt = torch.max(a[..., :2], b[..., :2])
+    rb = torch.min(a[..., 2:], b[..., 2:])
+    inter = (rb - lt).clamp(min=0).prod(-1)
+    area_a = (a[..., 2:] - a[..., :2]).clamp(min=0).prod(-1)
+    area_b = (b[..., 2:] - b[..., :2]).clamp(min=0).prod(-1)
+    return inter / (area_a + area_b - inter).clamp(min=1e-9)
+
+
+def _average_precision(scores, labels):
+    """How well the confidence ranks cells with a face above empty cells (1 = perfect)."""
+    order = scores.argsort(descending=True)
+    hits = labels[order].float()
+    precision = hits.cumsum(0) / torch.arange(1, len(hits) + 1, device=hits.device)
+    return ((precision * hits).sum() / hits.sum().clamp(min=1)).item()
+
+
+def detection_metrics(preds, targets, conf_threshold=0.5, iou_threshold=0.5):
+    obj = targets[..., 4] == 1
+    conf = preds[..., 4]
+    ious = _iou(_cell_boxes(preds), _cell_boxes(targets))
+
+    detected = conf > conf_threshold
+    correct = (detected & obj & (ious > iou_threshold)).sum().item()
+    n_obj = obj.sum().item()
+    return {
+        "conf_ap": _average_precision(conf.flatten(), obj.flatten()),
+        "box_iou": ious[obj].mean().item() if n_obj else 0.0,
+        "precision": correct / max(detected.sum().item(), 1),
+        "recall": correct / max(n_obj, 1),
+    }

api/src/yolo/tasks.py

@@ -10,12 +10,15 @@ from torchvision.utils import draw_bounding_boxes
 
 from src.common.checkpoints import checkpoint_path, load_checkpoint
 from src.common.device import get_device
-from src.common.training import benchmark_num_workers, fit
+from src.common.training import adamw, benchmark_num_workers, fit, warmup_cosine
 from src.common.visualize import show_image
 from src.models.yolo import Yolo_model, convert_prediction
 from src.yolo.config import Config
 from src.yolo.dataset import YoloDataset
 from src.yolo.loss import YoloLoss
+from src.yolo.metrics import detection_metrics
+from src.yolo.merge import merge_coco
+from src.yolo.wider import convert_wider
 
 
 def build_dataset(cfg: Config, folder: str, augment: bool):
@@ -50,7 +53,7 @@ def draw_prediction(label, image, threshold):
 
 
 def train(cfg: Config):
-    """Train the detector; saves the model with the lowest validation loss."""
+    """Train the detector; saves the model with the highest validation conf_ap."""
     device = get_device(cfg.device)
     train_loader = DataLoader(build_dataset(cfg, cfg.train_dir, cfg.augment_train),
                               batch_size=cfg.batch_size, shuffle=True, num_workers=cfg.train_workers)
@@ -58,11 +61,13 @@ def train(cfg: Config):
                             batch_size=cfg.batch_size, shuffle=False, num_workers=cfg.val_workers)
 
     model = build_model(cfg).to(device)
-    optimizer = torch.optim.Adam(model.parameters(), lr=cfg.learning_rate, weight_decay=cfg.weight_decay)
+    optimizer = adamw(model, cfg.learning_rate, cfg.weight_decay)
+    scheduler = warmup_cosine(optimizer, cfg.epochs, cfg.warmup_epochs, cfg.min_learning_rate)
 
     fit(model, YoloLoss(), optimizer, train_loader, val_loader, device,
         epochs=cfg.epochs, save_dir=cfg.save_dir, model_name=cfg.model_name, save=cfg.save,
-        monitor="loss")
+        save_every_epoch=cfg.save_every_epoch,
+        monitor="conf_ap", val_metrics=detection_metrics, config=cfg, scheduler=scheduler)
 
 
 def view_data(cfg: Config):
@@ -146,4 +151,6 @@ TASKS = {
     "sample": sample,
     "benchmark-workers": benchmark_workers,
     "webcam": webcam,
+    "convert-wider": convert_wider,
+    "merge-coco": merge_coco,
 }

api/src/yolo/wider.py

@@ -0,0 +1,135 @@
+"""Convert the WIDER FACE dataset to the COCO folders YoloDataset reads.
+
+WIDER_train becomes <wider_out_dir>/train and WIDER_val becomes
+<wider_out_dir>/test (WIDER's own test split has no public boxes). Each folder
+gets its images (copied, flat) and <annotation_file> with one category,
+1 = face, like faces_scenery.
+
+Most WIDER faces are tiny faces in crowds, which the model cannot use, so:
+- faces smaller than --wider-min-face px after resizing to --image-size are dropped,
+- images where two faces fall in the same --grid cell are skipped. YoloDataset
+  keeps one face per cell, so the other face would be trained as background.
+"""
+
+import json
+import shutil
+from pathlib import Path
+
+import cv2
+from tqdm import tqdm
+
+from src.common.boxes import grid_cell
+from src.yolo.config import Config
+
+# WIDER split -> output folder (train_dir / val_dir)
+SPLITS = {"train": "train", "val": "test"}
+
+
+def find(root: Path, pattern: str) -> Path:
+    """First match of pattern below root, so nested unzips (WIDER_train/WIDER_train/...) work too."""
+    matches = sorted(root.rglob(pattern))
+    if not matches:
+        raise FileNotFoundError(f"{pattern} not found under {root}")
+    return matches[0]
+
+
+def parse_ground_truth(path: Path):
+    """Yield (image path, faces), one face per row as
+    [x, y, w, h, blur, expression, illumination, invalid, occlusion, pose]."""
+    lines = [line.strip() for line in path.read_text().splitlines() if line.strip()]
+    i = 0
+    while i < len(lines):
+        name, count = lines[i], int(lines[i + 1])
+        faces = [[int(v) for v in line.split()] for line in lines[i + 2 : i + 2 + count]]
+        i += 2 + max(count, 1)  # an image without faces still has one "0 0 0 ..." row
+        yield name, faces
+
+
+def usable_boxes(faces, img_w, img_h, image_size, min_face):
+    """WIDER faces -> COCO [xmin, ymin, w, h] boxes in whole pixels, clipped to
+    the image and at least min_face px wide and high once resized to image_size."""
+    boxes = []
+    for x, y, w, h, _blur, _expression, _illumination, invalid, _occlusion, _pose in faces:
+        x0, y0 = max(x, 0), max(y, 0)
+        w, h = min(x + w, img_w) - x0, min(y + h, img_h) - y0
+        if invalid or w < 1 or h < 1:
+            continue
+        if w * image_size / img_w < min_face or h * image_size / img_h < min_face:
+            continue
+        boxes.append([x0, y0, w, h])
+    return boxes
+
+
+def shares_grid_cell(boxes, img_w, img_h, image_size, grid, eps=1e-3):
+    """True when two box centres can land in the same grid cell after resizing
+    to image_size. A centre within eps px of a cell border counts for both
+    cells, since Albumentations' float32 math can round it to either side."""
+    sx, sy = image_size / img_w, image_size / img_h
+    seen = set()
+    for x, y, w, h in boxes:
+        cx, cy = (x + w / 2) * sx, (y + h / 2) * sy
+        cells = {grid_cell(cx + dx, cy + dy, image_size, image_size, grid)
+                 for dx in (-eps, eps) for dy in (-eps, eps)}
+        if cells & seen:
+            return True
+        seen |= cells
+    return False
+
+
+def convert_split(images_dir: Path, gt_file: Path, out_dir: Path, cfg: Config):
+    out_dir.mkdir(parents=True)
+    images, annotations = [], []
+    skipped = {"unreadable": 0, "faces sharing a cell": 0}
+
+    for name, faces in tqdm(list(parse_ground_truth(gt_file)), desc=out_dir.name):
+        source = images_dir / name
+        # read with OpenCV like YoloDataset does, so every kept image is loadable
+        image = cv2.imread(str(source))
+        if image is None:
+            skipped["unreadable"] += 1
+            continue
+        img_h, img_w = image.shape[:2]
+
+        boxes = usable_boxes(faces, img_w, img_h, cfg.image_size, cfg.wider_min_face)
+        if shares_grid_cell(boxes, img_w, img_h, cfg.image_size, cfg.grid):
+            skipped["faces sharing a cell"] += 1
+            continue
+
+        target = out_dir / source.name
+        if target.exists():
+            raise FileExistsError(f"two WIDER images are named {source.name}")
+        shutil.copy2(source, target)
+
+        image_id = len(images) + 1
+        images.append({"id": image_id, "file_name": source.name, "width": img_w, "height": img_h})
+        for box in boxes:
+            annotations.append({"id": len(annotations) + 1, "image_id": image_id, "category_id": 1,
+                                "bbox": box, "area": box[2] * box[3], "iscrowd": 0})
+
+    coco = {"images": images, "annotations": annotations,
+            "categories": [{"id": 1, "name": "face", "supercategory": ""}]}
+    (out_dir / cfg.annotation_file).write_text(json.dumps(coco))
+
+    empty = len(images) - len({a["image_id"] for a in annotations})
+    print(f"{out_dir}: {len(images)} images ({empty} without faces), {len(annotations)} faces. "
+          f"Skipped: " + ", ".join(f"{n} {reason}" for reason, n in skipped.items()))
+
+
+def convert_wider(cfg: Config):
+    """Convert WIDER FACE (--wider-dir) to COCO train/ and test/ folders in --wider-out-dir."""
+    if not cfg.wider_dir:
+        raise SystemExit("Set --wider-dir to the folder with WIDER_train/, WIDER_val/ and wider_face_split/")
+    root, out = Path(cfg.wider_dir), Path(cfg.wider_out_dir)
+
+    sources = {split: (find(root, f"WIDER_{split}/images"), find(root, f"wider_face_{split}_bbx_gt.txt"))
+               for split in SPLITS}
+    for folder in SPLITS.values():
+        if (out / folder).exists():
+            raise SystemExit(f"{out / folder} already exists, delete it or pick another --wider-out-dir")
+
+    for split, folder in SPLITS.items():
+        images_dir, gt_file = sources[split]
+        convert_split(images_dir, gt_file, out / folder, cfg)
+
+    print(f"Train with: --train-dir {out / 'train'} --val-dir {out / 'test'} "
+          f"--image-size {cfg.image_size} --grid {cfg.grid} --labels 1")

api/train_test.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+
+for ((c = 1; c <= 32; c *= 2)); do
+    python -m src.main yolo.train --train-dir ./data/faces_combined/train/ --val-dir ./data/faces_combined/test/ \
+        --no-augment-val --c-hidden "$c" --batch-size 64 --epochs 200 --run-name "bn-combined-faces-c$c" \
+        --no-save --train-workers 4 --image-size 256 --grid 8 \
+        --learning-rate 1e-3 --weight-decay 0.05 --warmup-epochs 3 --min-learning-rate 1e-5
+done
+
+
+sudo -n shutdown -h +2

caddy/Caddyfile

@@ -13,10 +13,6 @@
 	log
 }
 
-www.marvinkrausser.com {
-	redir https://marvinkrausser.com{uri} permanent
-}
-
 marvinkrausser.com {
 	import common
 	reverse_proxy cnn_website:8080
@@ -60,16 +56,21 @@ game.marvinkrausser.com {
 		reverse_proxy wizard_website:8082
 	}
 
+	# handle_path takes a single path, so use named matchers to also match the
+	# bare prefix. The prefixes are stripped, so the backends see "/..." only.
+	@actionblocks_website path /actionblocks/website /actionblocks/website/*
+	handle @actionblocks_website {
+		uri strip_prefix /actionblocks/website
+		reverse_proxy actionblocks-website:80
+	}
+
+	@actionblocks_server path /actionblocks/server /actionblocks/server/*
+	handle @actionblocks_server {
+		uri strip_prefix /actionblocks/server
+		reverse_proxy actionblocks-server:8080
+	}
+
 	handle {
 		respond 404
 	}
 }
-
-cloud.marvinkrausser.com {
-    reverse_proxy nextcloud:80
-
-    redir /.well-known/carddav /remote.php/dav 301
-    redir /.well-known/caldav /remote.php/dav 301
-
-    header Strict-Transport-Security "max-age=15552000; includeSubDomains"
-}

compose.yaml

@@ -23,7 +23,8 @@ services:
       - portfolio_network
       - wizard_website
       - wizard_api
-      - cloud_network
+      - actionBlocks_website
+      - actionBlocks_server
     pull_policy: never
     container_name: caddy_proxy
 
@@ -78,5 +79,7 @@ networks:
     name: "wizard_website"
   wizard_api:
     name: "wizard_api"
-  cloud_network:
+  actionBlocks_website:
-    name: "cloud_network"
+    name: "actionBlocks_website"
+  actionBlocks_server:
+    name: "actionBlocks_server"