added new metrics

api/analytics/run_viewer.html

@@ -396,6 +396,7 @@ function fmtParams(n) {
   return String(n);
 }
 const metricLabel = (m) => m.replace(/_/g, " ");
+const extraMetricNames = (runs) => [...new Set([...runs].flatMap((r) => Object.keys(r.summary.atBest)))];
 
 // ------------------------------------------------------------------ loading
 
@@ -497,6 +498,13 @@ function makeRun(dir, sourceId, info, metrics, architecture, diff) {
   const valLoss = col("val_loss"), valAcc = col("val_acc");
   const last = rows.length ? rows[rows.length - 1] : {};
   const result = info.result || {};
+  // Extra val metrics (e.g. YOLO's conf_ap): the value at the best epoch, whose weights were kept.
+  const bestRow = rows.find((r) => r.epoch === result.best_epoch) || last;
+  const atBest = {};
+  for (const c of metrics.columns) {
+    const m = c.match(/^val_(.+)$/);
+    if (m && m[1] !== "loss" && m[1] !== "acc" && typeof bestRow[c] === "number") atBest[m[1]] = bestRow[c];
+  }
   return {
     key: dir,
     source: sourceId,
@@ -519,6 +527,7 @@ function makeRun(dir, sourceId, info, metrics, architecture, diff) {
       finalTrainLoss: typeof last.train_loss === "number" ? last.train_loss : null,
       bestEpoch: result.best_epoch ?? null,
       monitor: result.monitor || "loss",
+      atBest,
       duration: result.duration_s ?? null,
       params: (info.model || {}).params ?? null,
       archHash: (info.model || {}).architecture_hash || null,
@@ -676,6 +685,10 @@ function runsColumns(varying) {
     { id: "minValLoss", label: "Min val loss", get: (r) => r.summary.minValLoss, num: true },
   ];
   if (hasAcc) cols.push({ id: "maxValAcc", label: "Max val acc", get: (r) => r.summary.maxValAcc, num: true });
+  for (const name of extraMetricNames(state.runs.values())) {
+    cols.push({ id: "best:" + name, label: metricLabel(name), title: `val ${metricLabel(name)} at the best epoch`,
+      get: (r) => r.summary.atBest[name], num: true });
+  }
   cols.push(
     { id: "bestEpoch", label: "Best epoch", get: (r) => r.summary.bestEpoch, num: true },
     { id: "duration", label: "Duration", get: (r) => r.summary.duration, num: true },
@@ -728,7 +741,7 @@ function renderRunsTable() {
     const th = h("th", {
       class: [sortable ? "sortable" : "", c.num ? "num" : "", c.cfg ? "cfg" : ""].join(" ").trim(),
       scope: "col",
-      title: c.cfg ? "setting that differs between the listed runs" : null,
+      title: c.title || (c.cfg ? "setting that differs between the listed runs" : null),
       "aria-sort": state.sort.col === c.id ? (state.sort.dir > 0 ? "ascending" : "descending") : null,
     }, c.label, arrow ? h("span", { class: "arrow" }, arrow) : null);
     if (sortable) th.addEventListener("click", () => {
@@ -1017,13 +1030,16 @@ function renderScatter() {
   const prevX = xSel.value || recall("scatterX");
   xSel.replaceChildren(...xOptions.map((o) => h("option", { value: o.id }, o.label)));
   xSel.value = xOptions.some((o) => o.id === prevX) ? prevX : xOptions[0].id;
-  const yOptions = Y_MEASURES.filter((mm) => runs.some((r) => mm.get(r) != null));
+  const measures = [...Y_MEASURES.slice(0, 2),
+    ...extraMetricNames(runs).map((name) => ({ id: "best:" + name, label: `val ${metricLabel(name)} at best epoch`, get: (r) => r.summary.atBest[name] })),
+    ...Y_MEASURES.slice(2)];
+  const yOptions = measures.filter((mm) => runs.some((r) => mm.get(r) != null));
   const prevY = ySel.value || recall("scatterY");
   ySel.replaceChildren(...yOptions.map((o) => h("option", { value: o.id }, o.label)));
   ySel.value = yOptions.some((o) => o.id === prevY) ? prevY : (yOptions[0] || {}).id || "";
 
   const xo = xOptions.find((o) => o.id === xSel.value);
-  const yo = Y_MEASURES.find((o) => o.id === ySel.value);
+  const yo = measures.find((o) => o.id === ySel.value);
   const container = $("s-chart");
   if (!xo || !yo) { container.replaceChildren(h("div", { class: "empty" }, "Nothing to plot.")); return; }
 
@@ -1131,6 +1147,7 @@ function renderCompare() {
     const f = flatten({ ...r.info, git: { ...(r.info.git || {}), changed_files: ((r.info.git || {}).changed_files || []).length } }, "", {});
     f["summary.min_val_loss"] = r.summary.minValLoss;
     if (r.summary.maxValAcc != null) f["summary.max_val_acc"] = r.summary.maxValAcc;
+    for (const [name, v] of Object.entries(r.summary.atBest)) f[`summary.val_${name}_at_best_epoch`] = v;
     f["summary.duration"] = fmtDuration(r.summary.duration);
     return f;
   });

api/src/common/training.py

@@ -23,34 +23,48 @@ def _default_prepare(batch, device):
     return inputs.to(device), targets.to(device)
 
 
-def _run_epoch(model, loss_fn, loader, device, prepare_batch, track_accuracy, optimizer=None, desc=""):
+def _run_epoch(model, loss_fn, loader, device, prepare_batch, track_accuracy, optimizer=None,
+               val_metrics=None, desc=""):
     training = optimizer is not None
     model.train(training)
 
     loss_sum, correct, count = 0.0, 0, 0
+    all_preds, all_targets = [], []
     with torch.set_grad_enabled(training):
         for batch in tqdm(loader, desc=desc, leave=False):
             inputs, targets = prepare_batch(batch, device)
 
             preds = model(inputs)
-            loss = loss_fn(preds, targets)
 
             if training:
+                loss = loss_fn(preds, targets)
                 optimizer.zero_grad()
                 loss.backward()
                 optimizer.step()
-
                 loss_sum += loss.item() * inputs.size(0)
+            else:
+                all_preds.append(preds)
+                all_targets.append(targets)
+
             count += inputs.size(0)
             if track_accuracy:
                 correct += (preds.argmax(dim=1) == targets).sum().item()
 
-    if torch.cuda.is_available():
+    if training:
-        torch.cuda.empty_cache()
-
         metrics = {"loss": loss_sum / max(count, 1)}
+    else:
+        # Validation loss over the whole set at once, so it doesn't depend on
+        # the batch size (YoloLoss averages per batch over the cells with a face).
+        preds, targets = torch.cat(all_preds), torch.cat(all_targets)
+        metrics = {"loss": loss_fn(preds, targets).item()}
+        if val_metrics:
+            metrics.update(val_metrics(preds, targets))
+        del preds, targets, all_preds, all_targets
     if track_accuracy:
         metrics["acc"] = correct / max(count, 1)
+
+    if torch.cuda.is_available():
+        torch.cuda.empty_cache()
     return metrics
 
 
@@ -65,9 +79,10 @@ def fit(
     save_dir: str,
     model_name: str,
     save: bool = True,
-    monitor: str = "loss",          # "loss" (lower is better) or "acc" (higher is better)
+    monitor: str = "loss",          # "loss" (lower is better) or any other val metric (higher is better)
     track_accuracy: bool = False,   # for classifiers
     prepare_batch=_default_prepare,
+    val_metrics=None,               # f(preds, targets) -> dict of extra metrics over the whole val set
     config=None,                    # the task's Config; enables the run log
 ):
     run = None
@@ -76,7 +91,7 @@ def fit(
                      train_loader, val_loader, device, monitor)
     try:
         best = _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epochs,
-                           save_dir, model_name, save, monitor, track_accuracy, prepare_batch, run)
+                           save_dir, model_name, save, monitor, track_accuracy, prepare_batch, val_metrics, run)
     except KeyboardInterrupt:
         if run:
             run.finish("interrupted")
@@ -91,7 +106,7 @@ def fit(
 
 
 def _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epochs,
-                save_dir, model_name, save, monitor, track_accuracy, prepare_batch, run):
+                save_dir, model_name, save, monitor, track_accuracy, prepare_batch, val_metrics, run):
     best = math.inf if monitor == "loss" else -math.inf
 
     for epoch in range(1, epochs + 1):
@@ -99,7 +114,7 @@ def _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epo
         train = _run_epoch(model, loss_fn, train_loader, device, prepare_batch, track_accuracy,
                            optimizer=optimizer, desc=f"Train {epoch}")
         val = _run_epoch(model, loss_fn, val_loader, device, prepare_batch, track_accuracy,
-                         desc=f"Validate {epoch}")
+                         val_metrics=val_metrics, desc=f"Validate {epoch}")
 
         improved = val[monitor] < best if monitor == "loss" else val[monitor] > best
         if improved:
@@ -116,6 +131,7 @@ def _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epo
         if track_accuracy:
             parts.append(f"train acc: {train['acc'] * 100:.1f}% | val acc: {val['acc'] * 100:.1f}%")
         parts.append(f"train loss: {train['loss']:.4f} | val loss: {val['loss']:.4f}")
+        parts += [f"val {k}: {v:.3f}" for k, v in val.items() if k not in ("loss", "acc")]
         if improved and save:
             parts.append("saved")
         print(" | ".join(parts))

api/src/models/yolo.py

@@ -32,9 +32,7 @@ class SkipBlock(nn.Module):
         )
 
     def forward(self, x):
-        # F.dropout defaults to training=True, so this stays active in eval
+        return F.leaky_relu(self.conv_skip(x) + self.conv(x), inplace=True)
-        # mode too. Kept as-is because the saved models were trained this way.
-        return F.dropout(F.leaky_relu(self.conv_skip(x) + self.conv(x), inplace=True), p=0.3)
 
 
 class Yolo_model(nn.Module):
@@ -75,7 +73,6 @@ class Yolo_model(nn.Module):
             nn.Conv2d(c_hidden * 4, c_hidden * 8, kernel_size=3, padding=1),
             nn.BatchNorm2d(c_hidden * 8),
             nn.LeakyReLU(inplace=True),
-            nn.Dropout(0.3),
 
             nn.AdaptiveAvgPool2d((grid, grid)),
             nn.Conv2d(c_hidden * 8, boxes * 5 + labels, kernel_size=1)

api/src/yolo/loss.py

@@ -41,7 +41,8 @@ class YoloLoss(nn.Module):
         ) if noobj_mask.any() else torch.tensor(0.0, device=predictions.device)
 
 
-        class_loss = F.binary_cross_entropy_with_logits(
+        # The model already applies a sigmoid, so these are probabilities.
+        class_loss = F.binary_cross_entropy(
             pred_classes[obj_mask],
             target_classes[obj_mask]
         ) if obj_mask.any() else torch.tensor(0.0, device=predictions.device)

api/src/yolo/metrics.py

@@ -0,0 +1,51 @@
+"""Detection metrics over the whole validation set, logged next to the loss.
+Per grid cell, like the loss: a cell is a detection when its confidence
+passes `conf_threshold`, and a correct one when the cell holds a face and
+the box overlaps the true box by at least `iou_threshold`."""
+
+import torch
+
+
+def _cell_boxes(t):
+    """[N, S, S, >=4] cell outputs (x, y relative to the cell, w, h relative to
+    the image) -> [N, S, S, 4] boxes as (xmin, ymin, xmax, ymax) in image units."""
+    S = t.shape[1]
+    i = torch.arange(S, device=t.device).view(S, 1)
+    j = torch.arange(S, device=t.device).view(1, S)
+    cx = (i + t[..., 0]) / S
+    cy = (j + t[..., 1]) / S
+    w, h = t[..., 2], t[..., 3]
+    return torch.stack([cx - w / 2, cy - h / 2, cx + w / 2, cy + h / 2], dim=-1)
+
+
+def _iou(a, b):
+    lt = torch.max(a[..., :2], b[..., :2])
+    rb = torch.min(a[..., 2:], b[..., 2:])
+    inter = (rb - lt).clamp(min=0).prod(-1)
+    area_a = (a[..., 2:] - a[..., :2]).clamp(min=0).prod(-1)
+    area_b = (b[..., 2:] - b[..., :2]).clamp(min=0).prod(-1)
+    return inter / (area_a + area_b - inter).clamp(min=1e-9)
+
+
+def _average_precision(scores, labels):
+    """How well the confidence ranks cells with a face above empty cells (1 = perfect)."""
+    order = scores.argsort(descending=True)
+    hits = labels[order].float()
+    precision = hits.cumsum(0) / torch.arange(1, len(hits) + 1, device=hits.device)
+    return ((precision * hits).sum() / hits.sum().clamp(min=1)).item()
+
+
+def detection_metrics(preds, targets, conf_threshold=0.5, iou_threshold=0.5):
+    obj = targets[..., 4] == 1
+    conf = preds[..., 4]
+    ious = _iou(_cell_boxes(preds), _cell_boxes(targets))
+
+    detected = conf > conf_threshold
+    correct = (detected & obj & (ious > iou_threshold)).sum().item()
+    n_obj = obj.sum().item()
+    return {
+        "conf_ap": _average_precision(conf.flatten(), obj.flatten()),
+        "box_iou": ious[obj].mean().item() if n_obj else 0.0,
+        "precision": correct / max(detected.sum().item(), 1),
+        "recall": correct / max(n_obj, 1),
+    }

api/src/yolo/tasks.py

@@ -16,6 +16,7 @@ from src.models.yolo import Yolo_model, convert_prediction
 from src.yolo.config import Config
 from src.yolo.dataset import YoloDataset
 from src.yolo.loss import YoloLoss
+from src.yolo.metrics import detection_metrics
 
 
 def build_dataset(cfg: Config, folder: str, augment: bool):
@@ -62,7 +63,7 @@ def train(cfg: Config):
 
     fit(model, YoloLoss(), optimizer, train_loader, val_loader, device,
         epochs=cfg.epochs, save_dir=cfg.save_dir, model_name=cfg.model_name, save=cfg.save,
-        monitor="loss", config=cfg)
+        monitor="loss", val_metrics=detection_metrics, config=cfg)
 
 
 def view_data(cfg: Config):

api/train_test.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+
+for ((c = 1; c <= 32; c *= 2)); do
+    python -m src.main yolo.train --train-dir ./data/faces_scenery/train/ --val-dir ./data/faces_scenery/test/ \
+        --no-augment-val --c-hidden "$c" --batch-size 64 --epochs 200 --run-name "bn-c$c" \
+        --no-save --train-workers 2 --image-size 64 --grid 6
+done
+
+
+sudo -n shutdown -h +2