marvin/CNN_Website
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added new metrics

Browse Source
This commit is contained in:
marvin
2026-10-03 18:59:01 +02:00
parent 35a3e03a10
commit 8edb0a1e55
7 changed files with 114 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/analytics/run_viewer.html
+20 -3
View File
@@ -396,6 +396,7 @@ function fmtParams(n) {
return String(n);
}
const metricLabel = (m) => m.replace(/_/g, " ");
const extraMetricNames = (runs) => [...new Set([...runs].flatMap((r) => Object.keys(r.summary.atBest)))];
// ------------------------------------------------------------------ loading
@@ -497,6 +498,13 @@ function makeRun(dir, sourceId, info, metrics, architecture, diff) {
const valLoss = col("val_loss"), valAcc = col("val_acc");
const last = rows.length ? rows[rows.length - 1] : {};
const result = info.result || {};
// Extra val metrics (e.g. YOLO's conf_ap): the value at the best epoch, whose weights were kept.
const bestRow = rows.find((r) => r.epoch === result.best_epoch) || last;
const atBest = {};
for (const c of metrics.columns) {
const m = c.match(/^val_(.+)$/);
if (m && m[1] !== "loss" && m[1] !== "acc" && typeof bestRow[c] === "number") atBest[m[1]] = bestRow[c];
}
return {
key: dir,
source: sourceId,
@@ -519,6 +527,7 @@ function makeRun(dir, sourceId, info, metrics, architecture, diff) {
finalTrainLoss: typeof last.train_loss === "number" ? last.train_loss : null,
bestEpoch: result.best_epoch ?? null,
monitor: result.monitor || "loss",
atBest,
duration: result.duration_s ?? null,
params: (info.model || {}).params ?? null,
archHash: (info.model || {}).architecture_hash || null,
@@ -676,6 +685,10 @@ function runsColumns(varying) {
{ id: "minValLoss", label: "Min val loss", get: (r) => r.summary.minValLoss, num: true },
];
if (hasAcc) cols.push({ id: "maxValAcc", label: "Max val acc", get: (r) => r.summary.maxValAcc, num: true });
for (const name of extraMetricNames(state.runs.values())) {
cols.push({ id: "best:" + name, label: metricLabel(name), title: `val ${metricLabel(name)} at the best epoch`,
get: (r) => r.summary.atBest[name], num: true });
}
cols.push(
{ id: "bestEpoch", label: "Best epoch", get: (r) => r.summary.bestEpoch, num: true },
{ id: "duration", label: "Duration", get: (r) => r.summary.duration, num: true },
@@ -728,7 +741,7 @@ function renderRunsTable() {
const th = h("th", {
class: [sortable ? "sortable" : "", c.num ? "num" : "", c.cfg ? "cfg" : ""].join(" ").trim(),
scope: "col",
title: c.cfg ? "setting that differs between the listed runs" : null,
title: c.title || (c.cfg ? "setting that differs between the listed runs" : null),
"aria-sort": state.sort.col === c.id ? (state.sort.dir > 0 ? "ascending" : "descending") : null,
}, c.label, arrow ? h("span", { class: "arrow" }, arrow) : null);
if (sortable) th.addEventListener("click", () => {
@@ -1017,13 +1030,16 @@ function renderScatter() {
const prevX = xSel.value || recall("scatterX");
xSel.replaceChildren(...xOptions.map((o) => h("option", { value: o.id }, o.label)));
xSel.value = xOptions.some((o) => o.id === prevX) ? prevX : xOptions[0].id;
const yOptions = Y_MEASURES.filter((mm) => runs.some((r) => mm.get(r) != null));
const measures = [...Y_MEASURES.slice(0, 2),
...extraMetricNames(runs).map((name) => ({ id: "best:" + name, label: `val ${metricLabel(name)} at best epoch`, get: (r) => r.summary.atBest[name] })),
...Y_MEASURES.slice(2)];
const yOptions = measures.filter((mm) => runs.some((r) => mm.get(r) != null));
const prevY = ySel.value || recall("scatterY");
ySel.replaceChildren(...yOptions.map((o) => h("option", { value: o.id }, o.label)));
ySel.value = yOptions.some((o) => o.id === prevY) ? prevY : (yOptions[0] || {}).id || "";
const xo = xOptions.find((o) => o.id === xSel.value);
const yo = Y_MEASURES.find((o) => o.id === ySel.value);
const yo = measures.find((o) => o.id === ySel.value);
const container = $("s-chart");
if (!xo || !yo) { container.replaceChildren(h("div", { class: "empty" }, "Nothing to plot.")); return; }
@@ -1131,6 +1147,7 @@ function renderCompare() {
const f = flatten({ ...r.info, git: { ...(r.info.git || {}), changed_files: ((r.info.git || {}).changed_files || []).length } }, "", {});
f["summary.min_val_loss"] = r.summary.minValLoss;
if (r.summary.maxValAcc != null) f["summary.max_val_acc"] = r.summary.maxValAcc;
for (const [name, v] of Object.entries(r.summary.atBest)) f[`summary.val_${name}_at_best_epoch`] = v;
f["summary.duration"] = fmtDuration(r.summary.duration);
return f;
});
api/src/common/training.py
+26 -10
View File
@@ -23,34 +23,48 @@ def _default_prepare(batch, device):
return inputs.to(device), targets.to(device)
def _run_epoch(model, loss_fn, loader, device, prepare_batch, track_accuracy, optimizer=None, desc=""):
def _run_epoch(model, loss_fn, loader, device, prepare_batch, track_accuracy, optimizer=None,
val_metrics=None, desc=""):
training = optimizer is not None
model.train(training)
loss_sum, correct, count = 0.0, 0, 0
all_preds, all_targets = [], []
with torch.set_grad_enabled(training):
for batch in tqdm(loader, desc=desc, leave=False):
inputs, targets = prepare_batch(batch, device)
preds = model(inputs)
loss = loss_fn(preds, targets)
if training:
loss = loss_fn(preds, targets)
optimizer.zero_grad()
loss.backward()
optimizer.step()
loss_sum += loss.item() * inputs.size(0)
else:
all_preds.append(preds)
all_targets.append(targets)
count += inputs.size(0)
if track_accuracy:
correct += (preds.argmax(dim=1) == targets).sum().item()
if torch.cuda.is_available():
torch.cuda.empty_cache()
if training:
metrics = {"loss": loss_sum / max(count, 1)}
else:
# Validation loss over the whole set at once, so it doesn't depend on
# the batch size (YoloLoss averages per batch over the cells with a face).
preds, targets = torch.cat(all_preds), torch.cat(all_targets)
metrics = {"loss": loss_fn(preds, targets).item()}
if val_metrics:
metrics.update(val_metrics(preds, targets))
del preds, targets, all_preds, all_targets
if track_accuracy:
metrics["acc"] = correct / max(count, 1)
if torch.cuda.is_available():
torch.cuda.empty_cache()
return metrics
@@ -65,9 +79,10 @@ def fit(
save_dir: str,
model_name: str,
save: bool = True,
monitor: str = "loss", # "loss" (lower is better) or "acc" (higher is better)
monitor: str = "loss", # "loss" (lower is better) or any other val metric (higher is better)
track_accuracy: bool = False, # for classifiers
prepare_batch=_default_prepare,
val_metrics=None, # f(preds, targets) -> dict of extra metrics over the whole val set
config=None, # the task's Config; enables the run log
):
run = None
@@ -76,7 +91,7 @@ def fit(
train_loader, val_loader, device, monitor)
try:
best = _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epochs,
save_dir, model_name, save, monitor, track_accuracy, prepare_batch, run)
save_dir, model_name, save, monitor, track_accuracy, prepare_batch, val_metrics, run)
except KeyboardInterrupt:
if run:
run.finish("interrupted")
@@ -91,7 +106,7 @@ def fit(
def _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epochs,
save_dir, model_name, save, monitor, track_accuracy, prepare_batch, run):
save_dir, model_name, save, monitor, track_accuracy, prepare_batch, val_metrics, run):
best = math.inf if monitor == "loss" else -math.inf
for epoch in range(1, epochs + 1):
@@ -99,7 +114,7 @@ def _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epo
train = _run_epoch(model, loss_fn, train_loader, device, prepare_batch, track_accuracy,
optimizer=optimizer, desc=f"Train {epoch}")
val = _run_epoch(model, loss_fn, val_loader, device, prepare_batch, track_accuracy,
desc=f"Validate {epoch}")
val_metrics=val_metrics, desc=f"Validate {epoch}")
improved = val[monitor] < best if monitor == "loss" else val[monitor] > best
if improved:
@@ -116,6 +131,7 @@ def _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epo
if track_accuracy:
parts.append(f"train acc: {train['acc'] * 100:.1f}% | val acc: {val['acc'] * 100:.1f}%")
parts.append(f"train loss: {train['loss']:.4f} | val loss: {val['loss']:.4f}")
parts += [f"val {k}: {v:.3f}" for k, v in val.items() if k not in ("loss", "acc")]
if improved and save:
parts.append("saved")
print(" | ".join(parts))
api/src/models/yolo.py
+1 -4
View File
@@ -32,9 +32,7 @@ class SkipBlock(nn.Module):
)
def forward(self, x):
# F.dropout defaults to training=True, so this stays active in eval
# mode too. Kept as-is because the saved models were trained this way.
return F.dropout(F.leaky_relu(self.conv_skip(x) + self.conv(x), inplace=True), p=0.3)
return F.leaky_relu(self.conv_skip(x) + self.conv(x), inplace=True)
class Yolo_model(nn.Module):
@@ -75,7 +73,6 @@ class Yolo_model(nn.Module):
nn.Conv2d(c_hidden * 4, c_hidden * 8, kernel_size=3, padding=1),
nn.BatchNorm2d(c_hidden * 8),
nn.LeakyReLU(inplace=True),
nn.Dropout(0.3),
nn.AdaptiveAvgPool2d((grid, grid)),
nn.Conv2d(c_hidden * 8, boxes * 5 + labels, kernel_size=1)
api/src/yolo/loss.py
+2 -1
View File
@@ -41,7 +41,8 @@ class YoloLoss(nn.Module):
) if noobj_mask.any() else torch.tensor(0.0, device=predictions.device)
class_loss = F.binary_cross_entropy_with_logits(
# The model already applies a sigmoid, so these are probabilities.
class_loss = F.binary_cross_entropy(
pred_classes[obj_mask],
target_classes[obj_mask]
) if obj_mask.any() else torch.tensor(0.0, device=predictions.device)
api/src/yolo/metrics.py
+51
View File
@@ -0,0 +1,51 @@
"""Detection metrics over the whole validation set, logged next to the loss.
Per grid cell, like the loss: a cell is a detection when its confidence
passes `conf_threshold`, and a correct one when the cell holds a face and
the box overlaps the true box by at least `iou_threshold`."""
import torch
def _cell_boxes(t):
"""[N, S, S, >=4] cell outputs (x, y relative to the cell, w, h relative to
the image) -> [N, S, S, 4] boxes as (xmin, ymin, xmax, ymax) in image units."""
S = t.shape[1]
i = torch.arange(S, device=t.device).view(S, 1)
j = torch.arange(S, device=t.device).view(1, S)
cx = (i + t[..., 0]) / S
cy = (j + t[..., 1]) / S
w, h = t[..., 2], t[..., 3]
return torch.stack([cx - w / 2, cy - h / 2, cx + w / 2, cy + h / 2], dim=-1)
def _iou(a, b):
lt = torch.max(a[..., :2], b[..., :2])
rb = torch.min(a[..., 2:], b[..., 2:])
inter = (rb - lt).clamp(min=0).prod(-1)
area_a = (a[..., 2:] - a[..., :2]).clamp(min=0).prod(-1)
area_b = (b[..., 2:] - b[..., :2]).clamp(min=0).prod(-1)
return inter / (area_a + area_b - inter).clamp(min=1e-9)
def _average_precision(scores, labels):
"""How well the confidence ranks cells with a face above empty cells (1 = perfect)."""
order = scores.argsort(descending=True)
hits = labels[order].float()
precision = hits.cumsum(0) / torch.arange(1, len(hits) + 1, device=hits.device)
return ((precision * hits).sum() / hits.sum().clamp(min=1)).item()
def detection_metrics(preds, targets, conf_threshold=0.5, iou_threshold=0.5):
obj = targets[..., 4] == 1
conf = preds[..., 4]
ious = _iou(_cell_boxes(preds), _cell_boxes(targets))
detected = conf > conf_threshold
correct = (detected & obj & (ious > iou_threshold)).sum().item()
n_obj = obj.sum().item()
return {
"conf_ap": _average_precision(conf.flatten(), obj.flatten()),
"box_iou": ious[obj].mean().item() if n_obj else 0.0,
"precision": correct / max(detected.sum().item(), 1),
"recall": correct / max(n_obj, 1),
}
api/src/yolo/tasks.py
+2 -1
View File
@@ -16,6 +16,7 @@ from src.models.yolo import Yolo_model, convert_prediction
from src.yolo.config import Config
from src.yolo.dataset import YoloDataset
from src.yolo.loss import YoloLoss
from src.yolo.metrics import detection_metrics
def build_dataset(cfg: Config, folder: str, augment: bool):
@@ -62,7 +63,7 @@ def train(cfg: Config):
fit(model, YoloLoss(), optimizer, train_loader, val_loader, device,
epochs=cfg.epochs, save_dir=cfg.save_dir, model_name=cfg.model_name, save=cfg.save,
monitor="loss", config=cfg)
monitor="loss", val_metrics=detection_metrics, config=cfg)
def view_data(cfg: Config):
api/train_test.sh
+11
View File
@@ -0,0 +1,11 @@
#!/usr/bin/env bash
for ((c = 1; c <= 32; c *= 2)); do
python -m src.main yolo.train --train-dir ./data/faces_scenery/train/ --val-dir ./data/faces_scenery/test/ \
--no-augment-val --c-hidden "$c" --batch-size 64 --epochs 200 --run-name "bn-c$c" \
--no-save --train-workers 2 --image-size 64 --grid 6
done
sudo -n shutdown -h +2