diff --git a/api/analytics/run_viewer.html b/api/analytics/run_viewer.html index 1d9c997..4daeffa 100644 --- a/api/analytics/run_viewer.html +++ b/api/analytics/run_viewer.html @@ -396,6 +396,7 @@ function fmtParams(n) { return String(n); } const metricLabel = (m) => m.replace(/_/g, " "); +const extraMetricNames = (runs) => [...new Set([...runs].flatMap((r) => Object.keys(r.summary.atBest)))]; // ------------------------------------------------------------------ loading @@ -497,6 +498,13 @@ function makeRun(dir, sourceId, info, metrics, architecture, diff) { const valLoss = col("val_loss"), valAcc = col("val_acc"); const last = rows.length ? rows[rows.length - 1] : {}; const result = info.result || {}; + // Extra val metrics (e.g. YOLO's conf_ap): the value at the best epoch, whose weights were kept. + const bestRow = rows.find((r) => r.epoch === result.best_epoch) || last; + const atBest = {}; + for (const c of metrics.columns) { + const m = c.match(/^val_(.+)$/); + if (m && m[1] !== "loss" && m[1] !== "acc" && typeof bestRow[c] === "number") atBest[m[1]] = bestRow[c]; + } return { key: dir, source: sourceId, @@ -519,6 +527,7 @@ function makeRun(dir, sourceId, info, metrics, architecture, diff) { finalTrainLoss: typeof last.train_loss === "number" ? last.train_loss : null, bestEpoch: result.best_epoch ?? null, monitor: result.monitor || "loss", + atBest, duration: result.duration_s ?? null, params: (info.model || {}).params ?? null, archHash: (info.model || {}).architecture_hash || null, @@ -676,6 +685,10 @@ function runsColumns(varying) { { id: "minValLoss", label: "Min val loss", get: (r) => r.summary.minValLoss, num: true }, ]; if (hasAcc) cols.push({ id: "maxValAcc", label: "Max val acc", get: (r) => r.summary.maxValAcc, num: true }); + for (const name of extraMetricNames(state.runs.values())) { + cols.push({ id: "best:" + name, label: metricLabel(name), title: `val ${metricLabel(name)} at the best epoch`, + get: (r) => r.summary.atBest[name], num: true }); + } cols.push( { id: "bestEpoch", label: "Best epoch", get: (r) => r.summary.bestEpoch, num: true }, { id: "duration", label: "Duration", get: (r) => r.summary.duration, num: true }, @@ -728,7 +741,7 @@ function renderRunsTable() { const th = h("th", { class: [sortable ? "sortable" : "", c.num ? "num" : "", c.cfg ? "cfg" : ""].join(" ").trim(), scope: "col", - title: c.cfg ? "setting that differs between the listed runs" : null, + title: c.title || (c.cfg ? "setting that differs between the listed runs" : null), "aria-sort": state.sort.col === c.id ? (state.sort.dir > 0 ? "ascending" : "descending") : null, }, c.label, arrow ? h("span", { class: "arrow" }, arrow) : null); if (sortable) th.addEventListener("click", () => { @@ -1017,13 +1030,16 @@ function renderScatter() { const prevX = xSel.value || recall("scatterX"); xSel.replaceChildren(...xOptions.map((o) => h("option", { value: o.id }, o.label))); xSel.value = xOptions.some((o) => o.id === prevX) ? prevX : xOptions[0].id; - const yOptions = Y_MEASURES.filter((mm) => runs.some((r) => mm.get(r) != null)); + const measures = [...Y_MEASURES.slice(0, 2), + ...extraMetricNames(runs).map((name) => ({ id: "best:" + name, label: `val ${metricLabel(name)} at best epoch`, get: (r) => r.summary.atBest[name] })), + ...Y_MEASURES.slice(2)]; + const yOptions = measures.filter((mm) => runs.some((r) => mm.get(r) != null)); const prevY = ySel.value || recall("scatterY"); ySel.replaceChildren(...yOptions.map((o) => h("option", { value: o.id }, o.label))); ySel.value = yOptions.some((o) => o.id === prevY) ? prevY : (yOptions[0] || {}).id || ""; const xo = xOptions.find((o) => o.id === xSel.value); - const yo = Y_MEASURES.find((o) => o.id === ySel.value); + const yo = measures.find((o) => o.id === ySel.value); const container = $("s-chart"); if (!xo || !yo) { container.replaceChildren(h("div", { class: "empty" }, "Nothing to plot.")); return; } @@ -1131,6 +1147,7 @@ function renderCompare() { const f = flatten({ ...r.info, git: { ...(r.info.git || {}), changed_files: ((r.info.git || {}).changed_files || []).length } }, "", {}); f["summary.min_val_loss"] = r.summary.minValLoss; if (r.summary.maxValAcc != null) f["summary.max_val_acc"] = r.summary.maxValAcc; + for (const [name, v] of Object.entries(r.summary.atBest)) f[`summary.val_${name}_at_best_epoch`] = v; f["summary.duration"] = fmtDuration(r.summary.duration); return f; }); diff --git a/api/src/common/training.py b/api/src/common/training.py index d79811e..033d994 100644 --- a/api/src/common/training.py +++ b/api/src/common/training.py @@ -23,34 +23,48 @@ def _default_prepare(batch, device): return inputs.to(device), targets.to(device) -def _run_epoch(model, loss_fn, loader, device, prepare_batch, track_accuracy, optimizer=None, desc=""): +def _run_epoch(model, loss_fn, loader, device, prepare_batch, track_accuracy, optimizer=None, + val_metrics=None, desc=""): training = optimizer is not None model.train(training) loss_sum, correct, count = 0.0, 0, 0 + all_preds, all_targets = [], [] with torch.set_grad_enabled(training): for batch in tqdm(loader, desc=desc, leave=False): inputs, targets = prepare_batch(batch, device) preds = model(inputs) - loss = loss_fn(preds, targets) if training: + loss = loss_fn(preds, targets) optimizer.zero_grad() loss.backward() optimizer.step() + loss_sum += loss.item() * inputs.size(0) + else: + all_preds.append(preds) + all_targets.append(targets) - loss_sum += loss.item() * inputs.size(0) count += inputs.size(0) if track_accuracy: correct += (preds.argmax(dim=1) == targets).sum().item() - if torch.cuda.is_available(): - torch.cuda.empty_cache() - - metrics = {"loss": loss_sum / max(count, 1)} + if training: + metrics = {"loss": loss_sum / max(count, 1)} + else: + # Validation loss over the whole set at once, so it doesn't depend on + # the batch size (YoloLoss averages per batch over the cells with a face). + preds, targets = torch.cat(all_preds), torch.cat(all_targets) + metrics = {"loss": loss_fn(preds, targets).item()} + if val_metrics: + metrics.update(val_metrics(preds, targets)) + del preds, targets, all_preds, all_targets if track_accuracy: metrics["acc"] = correct / max(count, 1) + + if torch.cuda.is_available(): + torch.cuda.empty_cache() return metrics @@ -65,9 +79,10 @@ def fit( save_dir: str, model_name: str, save: bool = True, - monitor: str = "loss", # "loss" (lower is better) or "acc" (higher is better) + monitor: str = "loss", # "loss" (lower is better) or any other val metric (higher is better) track_accuracy: bool = False, # for classifiers prepare_batch=_default_prepare, + val_metrics=None, # f(preds, targets) -> dict of extra metrics over the whole val set config=None, # the task's Config; enables the run log ): run = None @@ -76,7 +91,7 @@ def fit( train_loader, val_loader, device, monitor) try: best = _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epochs, - save_dir, model_name, save, monitor, track_accuracy, prepare_batch, run) + save_dir, model_name, save, monitor, track_accuracy, prepare_batch, val_metrics, run) except KeyboardInterrupt: if run: run.finish("interrupted") @@ -91,7 +106,7 @@ def fit( def _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epochs, - save_dir, model_name, save, monitor, track_accuracy, prepare_batch, run): + save_dir, model_name, save, monitor, track_accuracy, prepare_batch, val_metrics, run): best = math.inf if monitor == "loss" else -math.inf for epoch in range(1, epochs + 1): @@ -99,7 +114,7 @@ def _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epo train = _run_epoch(model, loss_fn, train_loader, device, prepare_batch, track_accuracy, optimizer=optimizer, desc=f"Train {epoch}") val = _run_epoch(model, loss_fn, val_loader, device, prepare_batch, track_accuracy, - desc=f"Validate {epoch}") + val_metrics=val_metrics, desc=f"Validate {epoch}") improved = val[monitor] < best if monitor == "loss" else val[monitor] > best if improved: @@ -116,6 +131,7 @@ def _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epo if track_accuracy: parts.append(f"train acc: {train['acc'] * 100:.1f}% | val acc: {val['acc'] * 100:.1f}%") parts.append(f"train loss: {train['loss']:.4f} | val loss: {val['loss']:.4f}") + parts += [f"val {k}: {v:.3f}" for k, v in val.items() if k not in ("loss", "acc")] if improved and save: parts.append("saved") print(" | ".join(parts)) diff --git a/api/src/models/yolo.py b/api/src/models/yolo.py index b520227..98d1f8e 100644 --- a/api/src/models/yolo.py +++ b/api/src/models/yolo.py @@ -32,9 +32,7 @@ class SkipBlock(nn.Module): ) def forward(self, x): - # F.dropout defaults to training=True, so this stays active in eval - # mode too. Kept as-is because the saved models were trained this way. - return F.dropout(F.leaky_relu(self.conv_skip(x) + self.conv(x), inplace=True), p=0.3) + return F.leaky_relu(self.conv_skip(x) + self.conv(x), inplace=True) class Yolo_model(nn.Module): @@ -75,7 +73,6 @@ class Yolo_model(nn.Module): nn.Conv2d(c_hidden * 4, c_hidden * 8, kernel_size=3, padding=1), nn.BatchNorm2d(c_hidden * 8), nn.LeakyReLU(inplace=True), - nn.Dropout(0.3), nn.AdaptiveAvgPool2d((grid, grid)), nn.Conv2d(c_hidden * 8, boxes * 5 + labels, kernel_size=1) diff --git a/api/src/yolo/loss.py b/api/src/yolo/loss.py index b88f1ac..60fefe6 100644 --- a/api/src/yolo/loss.py +++ b/api/src/yolo/loss.py @@ -41,7 +41,8 @@ class YoloLoss(nn.Module): ) if noobj_mask.any() else torch.tensor(0.0, device=predictions.device) - class_loss = F.binary_cross_entropy_with_logits( + # The model already applies a sigmoid, so these are probabilities. + class_loss = F.binary_cross_entropy( pred_classes[obj_mask], target_classes[obj_mask] ) if obj_mask.any() else torch.tensor(0.0, device=predictions.device) diff --git a/api/src/yolo/metrics.py b/api/src/yolo/metrics.py new file mode 100644 index 0000000..816dafb --- /dev/null +++ b/api/src/yolo/metrics.py @@ -0,0 +1,51 @@ +"""Detection metrics over the whole validation set, logged next to the loss. +Per grid cell, like the loss: a cell is a detection when its confidence +passes `conf_threshold`, and a correct one when the cell holds a face and +the box overlaps the true box by at least `iou_threshold`.""" + +import torch + + +def _cell_boxes(t): + """[N, S, S, >=4] cell outputs (x, y relative to the cell, w, h relative to + the image) -> [N, S, S, 4] boxes as (xmin, ymin, xmax, ymax) in image units.""" + S = t.shape[1] + i = torch.arange(S, device=t.device).view(S, 1) + j = torch.arange(S, device=t.device).view(1, S) + cx = (i + t[..., 0]) / S + cy = (j + t[..., 1]) / S + w, h = t[..., 2], t[..., 3] + return torch.stack([cx - w / 2, cy - h / 2, cx + w / 2, cy + h / 2], dim=-1) + + +def _iou(a, b): + lt = torch.max(a[..., :2], b[..., :2]) + rb = torch.min(a[..., 2:], b[..., 2:]) + inter = (rb - lt).clamp(min=0).prod(-1) + area_a = (a[..., 2:] - a[..., :2]).clamp(min=0).prod(-1) + area_b = (b[..., 2:] - b[..., :2]).clamp(min=0).prod(-1) + return inter / (area_a + area_b - inter).clamp(min=1e-9) + + +def _average_precision(scores, labels): + """How well the confidence ranks cells with a face above empty cells (1 = perfect).""" + order = scores.argsort(descending=True) + hits = labels[order].float() + precision = hits.cumsum(0) / torch.arange(1, len(hits) + 1, device=hits.device) + return ((precision * hits).sum() / hits.sum().clamp(min=1)).item() + + +def detection_metrics(preds, targets, conf_threshold=0.5, iou_threshold=0.5): + obj = targets[..., 4] == 1 + conf = preds[..., 4] + ious = _iou(_cell_boxes(preds), _cell_boxes(targets)) + + detected = conf > conf_threshold + correct = (detected & obj & (ious > iou_threshold)).sum().item() + n_obj = obj.sum().item() + return { + "conf_ap": _average_precision(conf.flatten(), obj.flatten()), + "box_iou": ious[obj].mean().item() if n_obj else 0.0, + "precision": correct / max(detected.sum().item(), 1), + "recall": correct / max(n_obj, 1), + } diff --git a/api/src/yolo/tasks.py b/api/src/yolo/tasks.py index b01c1ca..ed33b70 100644 --- a/api/src/yolo/tasks.py +++ b/api/src/yolo/tasks.py @@ -16,6 +16,7 @@ from src.models.yolo import Yolo_model, convert_prediction from src.yolo.config import Config from src.yolo.dataset import YoloDataset from src.yolo.loss import YoloLoss +from src.yolo.metrics import detection_metrics def build_dataset(cfg: Config, folder: str, augment: bool): @@ -62,7 +63,7 @@ def train(cfg: Config): fit(model, YoloLoss(), optimizer, train_loader, val_loader, device, epochs=cfg.epochs, save_dir=cfg.save_dir, model_name=cfg.model_name, save=cfg.save, - monitor="loss", config=cfg) + monitor="loss", val_metrics=detection_metrics, config=cfg) def view_data(cfg: Config): diff --git a/api/train_test.sh b/api/train_test.sh new file mode 100644 index 0000000..54ad4e5 --- /dev/null +++ b/api/train_test.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + + +for ((c = 1; c <= 32; c *= 2)); do + python -m src.main yolo.train --train-dir ./data/faces_scenery/train/ --val-dir ./data/faces_scenery/test/ \ + --no-augment-val --c-hidden "$c" --batch-size 64 --epochs 200 --run-name "bn-c$c" \ + --no-save --train-workers 2 --image-size 64 --grid 6 +done + + +sudo -n shutdown -h +2