diff --git a/api/analytics/run_viewer.html b/api/analytics/run_viewer.html
index 1d9c997..4daeffa 100644
--- a/api/analytics/run_viewer.html
+++ b/api/analytics/run_viewer.html
@@ -396,6 +396,7 @@ function fmtParams(n) {
return String(n);
}
const metricLabel = (m) => m.replace(/_/g, " ");
+const extraMetricNames = (runs) => [...new Set([...runs].flatMap((r) => Object.keys(r.summary.atBest)))];
// ------------------------------------------------------------------ loading
@@ -497,6 +498,13 @@ function makeRun(dir, sourceId, info, metrics, architecture, diff) {
const valLoss = col("val_loss"), valAcc = col("val_acc");
const last = rows.length ? rows[rows.length - 1] : {};
const result = info.result || {};
+ // Extra val metrics (e.g. YOLO's conf_ap): the value at the best epoch, whose weights were kept.
+ const bestRow = rows.find((r) => r.epoch === result.best_epoch) || last;
+ const atBest = {};
+ for (const c of metrics.columns) {
+ const m = c.match(/^val_(.+)$/);
+ if (m && m[1] !== "loss" && m[1] !== "acc" && typeof bestRow[c] === "number") atBest[m[1]] = bestRow[c];
+ }
return {
key: dir,
source: sourceId,
@@ -519,6 +527,7 @@ function makeRun(dir, sourceId, info, metrics, architecture, diff) {
finalTrainLoss: typeof last.train_loss === "number" ? last.train_loss : null,
bestEpoch: result.best_epoch ?? null,
monitor: result.monitor || "loss",
+ atBest,
duration: result.duration_s ?? null,
params: (info.model || {}).params ?? null,
archHash: (info.model || {}).architecture_hash || null,
@@ -676,6 +685,10 @@ function runsColumns(varying) {
{ id: "minValLoss", label: "Min val loss", get: (r) => r.summary.minValLoss, num: true },
];
if (hasAcc) cols.push({ id: "maxValAcc", label: "Max val acc", get: (r) => r.summary.maxValAcc, num: true });
+ for (const name of extraMetricNames(state.runs.values())) {
+ cols.push({ id: "best:" + name, label: metricLabel(name), title: `val ${metricLabel(name)} at the best epoch`,
+ get: (r) => r.summary.atBest[name], num: true });
+ }
cols.push(
{ id: "bestEpoch", label: "Best epoch", get: (r) => r.summary.bestEpoch, num: true },
{ id: "duration", label: "Duration", get: (r) => r.summary.duration, num: true },
@@ -728,7 +741,7 @@ function renderRunsTable() {
const th = h("th", {
class: [sortable ? "sortable" : "", c.num ? "num" : "", c.cfg ? "cfg" : ""].join(" ").trim(),
scope: "col",
- title: c.cfg ? "setting that differs between the listed runs" : null,
+ title: c.title || (c.cfg ? "setting that differs between the listed runs" : null),
"aria-sort": state.sort.col === c.id ? (state.sort.dir > 0 ? "ascending" : "descending") : null,
}, c.label, arrow ? h("span", { class: "arrow" }, arrow) : null);
if (sortable) th.addEventListener("click", () => {
@@ -1017,13 +1030,16 @@ function renderScatter() {
const prevX = xSel.value || recall("scatterX");
xSel.replaceChildren(...xOptions.map((o) => h("option", { value: o.id }, o.label)));
xSel.value = xOptions.some((o) => o.id === prevX) ? prevX : xOptions[0].id;
- const yOptions = Y_MEASURES.filter((mm) => runs.some((r) => mm.get(r) != null));
+ const measures = [...Y_MEASURES.slice(0, 2),
+ ...extraMetricNames(runs).map((name) => ({ id: "best:" + name, label: `val ${metricLabel(name)} at best epoch`, get: (r) => r.summary.atBest[name] })),
+ ...Y_MEASURES.slice(2)];
+ const yOptions = measures.filter((mm) => runs.some((r) => mm.get(r) != null));
const prevY = ySel.value || recall("scatterY");
ySel.replaceChildren(...yOptions.map((o) => h("option", { value: o.id }, o.label)));
ySel.value = yOptions.some((o) => o.id === prevY) ? prevY : (yOptions[0] || {}).id || "";
const xo = xOptions.find((o) => o.id === xSel.value);
- const yo = Y_MEASURES.find((o) => o.id === ySel.value);
+ const yo = measures.find((o) => o.id === ySel.value);
const container = $("s-chart");
if (!xo || !yo) { container.replaceChildren(h("div", { class: "empty" }, "Nothing to plot.")); return; }
@@ -1131,6 +1147,7 @@ function renderCompare() {
const f = flatten({ ...r.info, git: { ...(r.info.git || {}), changed_files: ((r.info.git || {}).changed_files || []).length } }, "", {});
f["summary.min_val_loss"] = r.summary.minValLoss;
if (r.summary.maxValAcc != null) f["summary.max_val_acc"] = r.summary.maxValAcc;
+ for (const [name, v] of Object.entries(r.summary.atBest)) f[`summary.val_${name}_at_best_epoch`] = v;
f["summary.duration"] = fmtDuration(r.summary.duration);
return f;
});
diff --git a/api/src/common/training.py b/api/src/common/training.py
index d79811e..033d994 100644
--- a/api/src/common/training.py
+++ b/api/src/common/training.py
@@ -23,34 +23,48 @@ def _default_prepare(batch, device):
return inputs.to(device), targets.to(device)
-def _run_epoch(model, loss_fn, loader, device, prepare_batch, track_accuracy, optimizer=None, desc=""):
+def _run_epoch(model, loss_fn, loader, device, prepare_batch, track_accuracy, optimizer=None,
+ val_metrics=None, desc=""):
training = optimizer is not None
model.train(training)
loss_sum, correct, count = 0.0, 0, 0
+ all_preds, all_targets = [], []
with torch.set_grad_enabled(training):
for batch in tqdm(loader, desc=desc, leave=False):
inputs, targets = prepare_batch(batch, device)
preds = model(inputs)
- loss = loss_fn(preds, targets)
if training:
+ loss = loss_fn(preds, targets)
optimizer.zero_grad()
loss.backward()
optimizer.step()
+ loss_sum += loss.item() * inputs.size(0)
+ else:
+ all_preds.append(preds)
+ all_targets.append(targets)
- loss_sum += loss.item() * inputs.size(0)
count += inputs.size(0)
if track_accuracy:
correct += (preds.argmax(dim=1) == targets).sum().item()
- if torch.cuda.is_available():
- torch.cuda.empty_cache()
-
- metrics = {"loss": loss_sum / max(count, 1)}
+ if training:
+ metrics = {"loss": loss_sum / max(count, 1)}
+ else:
+ # Validation loss over the whole set at once, so it doesn't depend on
+ # the batch size (YoloLoss averages per batch over the cells with a face).
+ preds, targets = torch.cat(all_preds), torch.cat(all_targets)
+ metrics = {"loss": loss_fn(preds, targets).item()}
+ if val_metrics:
+ metrics.update(val_metrics(preds, targets))
+ del preds, targets, all_preds, all_targets
if track_accuracy:
metrics["acc"] = correct / max(count, 1)
+
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
return metrics
@@ -65,9 +79,10 @@ def fit(
save_dir: str,
model_name: str,
save: bool = True,
- monitor: str = "loss", # "loss" (lower is better) or "acc" (higher is better)
+ monitor: str = "loss", # "loss" (lower is better) or any other val metric (higher is better)
track_accuracy: bool = False, # for classifiers
prepare_batch=_default_prepare,
+ val_metrics=None, # f(preds, targets) -> dict of extra metrics over the whole val set
config=None, # the task's Config; enables the run log
):
run = None
@@ -76,7 +91,7 @@ def fit(
train_loader, val_loader, device, monitor)
try:
best = _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epochs,
- save_dir, model_name, save, monitor, track_accuracy, prepare_batch, run)
+ save_dir, model_name, save, monitor, track_accuracy, prepare_batch, val_metrics, run)
except KeyboardInterrupt:
if run:
run.finish("interrupted")
@@ -91,7 +106,7 @@ def fit(
def _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epochs,
- save_dir, model_name, save, monitor, track_accuracy, prepare_batch, run):
+ save_dir, model_name, save, monitor, track_accuracy, prepare_batch, val_metrics, run):
best = math.inf if monitor == "loss" else -math.inf
for epoch in range(1, epochs + 1):
@@ -99,7 +114,7 @@ def _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epo
train = _run_epoch(model, loss_fn, train_loader, device, prepare_batch, track_accuracy,
optimizer=optimizer, desc=f"Train {epoch}")
val = _run_epoch(model, loss_fn, val_loader, device, prepare_batch, track_accuracy,
- desc=f"Validate {epoch}")
+ val_metrics=val_metrics, desc=f"Validate {epoch}")
improved = val[monitor] < best if monitor == "loss" else val[monitor] > best
if improved:
@@ -116,6 +131,7 @@ def _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epo
if track_accuracy:
parts.append(f"train acc: {train['acc'] * 100:.1f}% | val acc: {val['acc'] * 100:.1f}%")
parts.append(f"train loss: {train['loss']:.4f} | val loss: {val['loss']:.4f}")
+ parts += [f"val {k}: {v:.3f}" for k, v in val.items() if k not in ("loss", "acc")]
if improved and save:
parts.append("saved")
print(" | ".join(parts))
diff --git a/api/src/models/yolo.py b/api/src/models/yolo.py
index b520227..98d1f8e 100644
--- a/api/src/models/yolo.py
+++ b/api/src/models/yolo.py
@@ -32,9 +32,7 @@ class SkipBlock(nn.Module):
)
def forward(self, x):
- # F.dropout defaults to training=True, so this stays active in eval
- # mode too. Kept as-is because the saved models were trained this way.
- return F.dropout(F.leaky_relu(self.conv_skip(x) + self.conv(x), inplace=True), p=0.3)
+ return F.leaky_relu(self.conv_skip(x) + self.conv(x), inplace=True)
class Yolo_model(nn.Module):
@@ -75,7 +73,6 @@ class Yolo_model(nn.Module):
nn.Conv2d(c_hidden * 4, c_hidden * 8, kernel_size=3, padding=1),
nn.BatchNorm2d(c_hidden * 8),
nn.LeakyReLU(inplace=True),
- nn.Dropout(0.3),
nn.AdaptiveAvgPool2d((grid, grid)),
nn.Conv2d(c_hidden * 8, boxes * 5 + labels, kernel_size=1)
diff --git a/api/src/yolo/loss.py b/api/src/yolo/loss.py
index b88f1ac..60fefe6 100644
--- a/api/src/yolo/loss.py
+++ b/api/src/yolo/loss.py
@@ -41,7 +41,8 @@ class YoloLoss(nn.Module):
) if noobj_mask.any() else torch.tensor(0.0, device=predictions.device)
- class_loss = F.binary_cross_entropy_with_logits(
+ # The model already applies a sigmoid, so these are probabilities.
+ class_loss = F.binary_cross_entropy(
pred_classes[obj_mask],
target_classes[obj_mask]
) if obj_mask.any() else torch.tensor(0.0, device=predictions.device)
diff --git a/api/src/yolo/metrics.py b/api/src/yolo/metrics.py
new file mode 100644
index 0000000..816dafb
--- /dev/null
+++ b/api/src/yolo/metrics.py
@@ -0,0 +1,51 @@
+"""Detection metrics over the whole validation set, logged next to the loss.
+Per grid cell, like the loss: a cell is a detection when its confidence
+passes `conf_threshold`, and a correct one when the cell holds a face and
+the box overlaps the true box by at least `iou_threshold`."""
+
+import torch
+
+
+def _cell_boxes(t):
+ """[N, S, S, >=4] cell outputs (x, y relative to the cell, w, h relative to
+ the image) -> [N, S, S, 4] boxes as (xmin, ymin, xmax, ymax) in image units."""
+ S = t.shape[1]
+ i = torch.arange(S, device=t.device).view(S, 1)
+ j = torch.arange(S, device=t.device).view(1, S)
+ cx = (i + t[..., 0]) / S
+ cy = (j + t[..., 1]) / S
+ w, h = t[..., 2], t[..., 3]
+ return torch.stack([cx - w / 2, cy - h / 2, cx + w / 2, cy + h / 2], dim=-1)
+
+
+def _iou(a, b):
+ lt = torch.max(a[..., :2], b[..., :2])
+ rb = torch.min(a[..., 2:], b[..., 2:])
+ inter = (rb - lt).clamp(min=0).prod(-1)
+ area_a = (a[..., 2:] - a[..., :2]).clamp(min=0).prod(-1)
+ area_b = (b[..., 2:] - b[..., :2]).clamp(min=0).prod(-1)
+ return inter / (area_a + area_b - inter).clamp(min=1e-9)
+
+
+def _average_precision(scores, labels):
+ """How well the confidence ranks cells with a face above empty cells (1 = perfect)."""
+ order = scores.argsort(descending=True)
+ hits = labels[order].float()
+ precision = hits.cumsum(0) / torch.arange(1, len(hits) + 1, device=hits.device)
+ return ((precision * hits).sum() / hits.sum().clamp(min=1)).item()
+
+
+def detection_metrics(preds, targets, conf_threshold=0.5, iou_threshold=0.5):
+ obj = targets[..., 4] == 1
+ conf = preds[..., 4]
+ ious = _iou(_cell_boxes(preds), _cell_boxes(targets))
+
+ detected = conf > conf_threshold
+ correct = (detected & obj & (ious > iou_threshold)).sum().item()
+ n_obj = obj.sum().item()
+ return {
+ "conf_ap": _average_precision(conf.flatten(), obj.flatten()),
+ "box_iou": ious[obj].mean().item() if n_obj else 0.0,
+ "precision": correct / max(detected.sum().item(), 1),
+ "recall": correct / max(n_obj, 1),
+ }
diff --git a/api/src/yolo/tasks.py b/api/src/yolo/tasks.py
index b01c1ca..ed33b70 100644
--- a/api/src/yolo/tasks.py
+++ b/api/src/yolo/tasks.py
@@ -16,6 +16,7 @@ from src.models.yolo import Yolo_model, convert_prediction
from src.yolo.config import Config
from src.yolo.dataset import YoloDataset
from src.yolo.loss import YoloLoss
+from src.yolo.metrics import detection_metrics
def build_dataset(cfg: Config, folder: str, augment: bool):
@@ -62,7 +63,7 @@ def train(cfg: Config):
fit(model, YoloLoss(), optimizer, train_loader, val_loader, device,
epochs=cfg.epochs, save_dir=cfg.save_dir, model_name=cfg.model_name, save=cfg.save,
- monitor="loss", config=cfg)
+ monitor="loss", val_metrics=detection_metrics, config=cfg)
def view_data(cfg: Config):
diff --git a/api/train_test.sh b/api/train_test.sh
new file mode 100644
index 0000000..54ad4e5
--- /dev/null
+++ b/api/train_test.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+
+for ((c = 1; c <= 32; c *= 2)); do
+ python -m src.main yolo.train --train-dir ./data/faces_scenery/train/ --val-dir ./data/faces_scenery/test/ \
+ --no-augment-val --c-hidden "$c" --batch-size 64 --epochs 200 --run-name "bn-c$c" \
+ --no-save --train-workers 2 --image-size 64 --grid 6
+done
+
+
+sudo -n shutdown -h +2