diff --git a/.gitignore b/.gitignore index e589450..37fdb03 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ api/data/ api/data.* api/saved_models/ +api/runs/ api/testimages/ **/__pycache__/ diff --git a/api/analytics/run_viewer.html b/api/analytics/run_viewer.html new file mode 100644 index 0000000..1d9c997 --- /dev/null +++ b/api/analytics/run_viewer.html @@ -0,0 +1,1343 @@ + + + + + +Training Run Viewer + + + + +
+
+
+

Training runs

+
No runs loaded
+
+
+ + + + + +
+ +
+

Load training logs

+

Open the api/runs folder, a series folder inside it, or a single run folder. + You can also drag one or more folders onto this page.

+

Each run folder needs a run.json; metrics.csv, + architecture.txt and changes.diff are used when present. + Model weights are never read.

+
+
+ +
+
+ + + + + +
+ +
+

Runs

+

Click rows to select up to 8 runs for the charts and comparisons below. + Columns marked ⚙ are settings that differ between the listed runs.

+
+
+ + + +
+
+ +
+

Training curves

+

Select runs above to plot them.

+
+ + + + +
+
+
+
+ Values per epoch (table) +
+
+
+ +
+

Settings vs. result

+

Every listed run as one point. Selected runs are coloured; the rest are grey.

+
+ + + +
+
+
+ +
+

Compare selected runs

+
+ +
+
+
+ +
+

Architecture

+

Model structure from architecture.txt: one run alone, or the difference between two.

+
+ + + +
+ 

+    

+
+    

+      
Code changes

+      
Uncommitted changes that were in the working tree when the run started (changes.diff).

+      

+        
+        
+      

+      

+    

+  

+

+
+
+
+
diff --git a/api/src/bird_cnn/config.py b/api/src/bird_cnn/config.py
index 17b439b..d9c284e 100644
--- a/api/src/bird_cnn/config.py
+++ b/api/src/bird_cnn/config.py
@@ -28,6 +28,10 @@ class Config:
     model_name: str = "bird_cnn"
     save: bool = True
 
+    # Run log (src/common/run_log.py): one folder per training run
+    runs_dir: str = "runs"            # empty = no run log
+    run_name: str = ""                # added to the run folder name, e.g. "bn-lr3e-4"
+
     # predict task
     image: str = "test1.jpg"
     checkpoint: str = ""                          # empty = //
diff --git a/api/src/bird_cnn/tasks.py b/api/src/bird_cnn/tasks.py
index 48dccfe..071e902 100644
--- a/api/src/bird_cnn/tasks.py
+++ b/api/src/bird_cnn/tasks.py
@@ -56,7 +56,7 @@ def train(cfg: Config):
 
     fit(model, nn.CrossEntropyLoss(), optimizer, train_loader, val_loader, device,
         epochs=cfg.epochs, save_dir=cfg.save_dir, model_name=cfg.model_name, save=cfg.save,
-        monitor="acc", track_accuracy=True)
+        monitor="acc", track_accuracy=True, config=cfg)
 
 
 def view_data(cfg: Config):
diff --git a/api/src/common/run_log.py b/api/src/common/run_log.py
new file mode 100644
index 0000000..f251276
--- /dev/null
+++ b/api/src/common/run_log.py
@@ -0,0 +1,197 @@
+"""Records every training run in its own folder, so runs with different
+settings or architectures can be compared later.
+
+    //[_]/
+        run.json          everything about the run: config, command, model
+                          (class, parameter counts, architecture hash),
+                          optimizer, loss, data sizes, environment, git state
+                          and the result (status, best epoch and metric,
+                          duration). Rewritten after every epoch.
+        metrics.csv       one row per epoch
+        architecture.txt  the model's module tree, str(model)
+        changes.diff      uncommitted code changes (only if there are any)
+        best.pt           state_dict of the best epoch so far
+
+result.status in run.json is "running" during training, then "completed",
+"failed" or "interrupted". A run whose process was killed stays "running".
+"""
+
+import csv
+import dataclasses
+import hashlib
+import json
+import os
+import platform
+import re
+import socket
+import subprocess
+import sys
+import time
+from datetime import datetime
+from pathlib import Path
+
+import torch
+
+REPO_DIR = Path(__file__).resolve().parent
+
+
+def _git(*args):
+    try:
+        return subprocess.run(["git", *args], cwd=REPO_DIR, capture_output=True, text=True,
+                              timeout=10, check=True).stdout
+    except (OSError, subprocess.SubprocessError):
+        return None
+
+
+def _jsonable(value):
+    if value is None or isinstance(value, (str, int, float, bool)):
+        return value
+    if isinstance(value, (list, tuple)):
+        return [_jsonable(v) for v in value]
+    if isinstance(value, dict):
+        return {str(k): _jsonable(v) for k, v in value.items()}
+    if isinstance(value, torch.Tensor):
+        return value.tolist() if value.numel() <= 16 else f"tensor{tuple(value.shape)}"
+    return str(value)
+
+
+def _module_settings(module):
+    """Public attributes and small buffers, e.g. a loss's reduction and class weights."""
+    settings = {k: v for k, v in vars(module).items() if not k.startswith("_") and k != "training"}
+    settings.update(dict(module.named_buffers()))
+    return _jsonable(settings)
+
+
+def _dataset_size(loader):
+    try:
+        return len(loader.dataset)
+    except TypeError:
+        return None
+
+
+class RunLog:
+    def __init__(self, runs_dir, model_name, config, model, optimizer, loss_fn,
+                 train_loader, val_loader, device, monitor):
+        self.monitor = monitor
+        self.started = time.time()
+        self.dir = self._make_dir(Path(runs_dir) / model_name, getattr(config, "run_name", ""))
+        self._csv_fields = None
+
+        architecture = str(model)
+        status = _git("status", "--porcelain")
+        diff = _git("diff", "HEAD")
+        gpu = torch.cuda.get_device_name(device) if torch.device(device).type == "cuda" else None
+
+        self.info = {
+            "id": f"{model_name}/{self.dir.name}",
+            "model_name": model_name,
+            "started": datetime.fromtimestamp(self.started).isoformat(timespec="seconds"),
+            "command": " ".join(sys.argv),
+            "config": _jsonable(dataclasses.asdict(config)),
+            "model": {
+                "class": type(model).__name__,
+                "params": sum(p.numel() for p in model.parameters()),
+                "trainable_params": sum(p.numel() for p in model.parameters() if p.requires_grad),
+                "architecture_hash": hashlib.sha256(architecture.encode()).hexdigest()[:12],
+            },
+            "optimizer": {
+                "class": type(optimizer).__name__,
+                **_jsonable({k: v for k, v in optimizer.param_groups[0].items() if k != "params"}),
+            },
+            "loss": {"class": type(loss_fn).__name__, **_module_settings(loss_fn)},
+            "data": {
+                "train_size": _dataset_size(train_loader),
+                "val_size": _dataset_size(val_loader),
+                "batch_size": train_loader.batch_size,
+                "train_workers": train_loader.num_workers,
+            },
+            "environment": {
+                "host": socket.gethostname(),
+                "platform": platform.platform(),
+                "python": platform.python_version(),
+                "torch": torch.__version__,
+                "cuda": torch.version.cuda,
+                "device": str(device),
+                "gpu": gpu,
+                "seed": torch.initial_seed(),
+            },
+            "git": {
+                "commit": (_git("rev-parse", "HEAD") or "").strip() or None,
+                "branch": (_git("rev-parse", "--abbrev-ref", "HEAD") or "").strip() or None,
+                "dirty": bool(status.strip()) if status is not None else None,
+                "changed_files": [line[3:] for line in status.splitlines()] if status else [],
+            },
+            "result": {
+                "status": "running",
+                "monitor": monitor,
+                "epochs_done": 0,
+                "best_epoch": None,
+                f"best_val_{monitor}": None,
+                "last": None,
+                "duration_s": 0,
+                "finished": None,
+                "error": None,
+            },
+        }
+
+        (self.dir / "architecture.txt").write_text(architecture + "\n")
+        if diff:
+            (self.dir / "changes.diff").write_text(diff)
+        self._write_info()
+        print(f"Logging run to {self.dir}")
+
+    @staticmethod
+    def _make_dir(parent, run_name):
+        name = datetime.now().strftime("%Y%m%d-%H%M%S")
+        if run_name:
+            name += "_" + re.sub(r"[^\w.-]+", "-", run_name)
+        parent.mkdir(parents=True, exist_ok=True)
+        for i in range(1, 100):
+            path = parent / (name if i == 1 else f"{name}-{i}")
+            try:
+                path.mkdir()
+                return path
+            except FileExistsError:
+                continue
+        raise FileExistsError(f"no free run folder for {parent / name}")
+
+    def _write_info(self):
+        # Write-then-rename so a reader never sees a half written file.
+        tmp = self.dir / "run.json.tmp"
+        tmp.write_text(json.dumps(self.info, indent=2) + "\n")
+        os.replace(tmp, self.dir / "run.json")
+
+    def log_epoch(self, epoch, train, val, lr, seconds, improved):
+        row = {"epoch": epoch,
+               **{f"train_{k}": v for k, v in train.items()},
+               **{f"val_{k}": v for k, v in val.items()},
+               "lr": lr, "seconds": round(seconds, 2), "improved": improved}
+        path = self.dir / "metrics.csv"
+        with path.open("a", newline="") as f:
+            if self._csv_fields is None:
+                self._csv_fields = list(row)
+                writer = csv.DictWriter(f, fieldnames=self._csv_fields)
+                writer.writeheader()
+            else:
+                writer = csv.DictWriter(f, fieldnames=self._csv_fields)
+            writer.writerow(row)
+
+        result = self.info["result"]
+        result["epochs_done"] = epoch
+        result["last"] = {k: v for k, v in row.items() if k != "improved"}
+        if improved:
+            result["best_epoch"] = epoch
+            result[f"best_val_{self.monitor}"] = val[self.monitor]
+        result["duration_s"] = round(time.time() - self.started, 1)
+        self._write_info()
+
+    def save_weights(self, model):
+        torch.save(model.state_dict(), self.dir / "best.pt")
+
+    def finish(self, status, error=None):
+        result = self.info["result"]
+        result["status"] = status
+        result["error"] = error
+        result["duration_s"] = round(time.time() - self.started, 1)
+        result["finished"] = datetime.now().isoformat(timespec="seconds")
+        self._write_info()
diff --git a/api/src/common/training.py b/api/src/common/training.py
index df24b56..dc19d90 100644
--- a/api/src/common/training.py
+++ b/api/src/common/training.py
@@ -2,7 +2,10 @@
 
 Each epoch trains, validates, prints the metrics and saves the model when the
 monitored metric improves. `prepare_batch` turns a loader batch into
-(inputs, targets); the default just moves both to the device."""
+(inputs, targets); the default just moves both to the device.
+
+Passing the task's `config` also records the run (settings, architecture,
+metrics per epoch, best weights) under config.runs_dir, see run_log.py."""
 
 import math
 import time
@@ -12,6 +15,7 @@ from torch.utils.data import DataLoader
 from tqdm import tqdm
 
 from src.common.checkpoints import save_checkpoint
+from src.common.run_log import RunLog
 
 
 def _default_prepare(batch, device):
@@ -64,10 +68,34 @@ def fit(
     monitor: str = "loss",          # "loss" (lower is better) or "acc" (higher is better)
     track_accuracy: bool = False,   # for classifiers
     prepare_batch=_default_prepare,
+    config=None,                    # the task's Config; enables the run log
 ):
+    run = None
+    if config is not None and config.runs_dir:
+        run = RunLog(config.runs_dir, model_name, config, model, optimizer, loss_fn,
+                     train_loader, val_loader, device, monitor)
+    try:
+        best = _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epochs,
+                           save_dir, model_name, save, monitor, track_accuracy, prepare_batch, run)
+    except KeyboardInterrupt:
+        if run:
+            run.finish("interrupted")
+        raise
+    except Exception as e:
+        if run:
+            run.finish("failed", error=repr(e))
+        raise
+    if run:
+        run.finish("completed")
+    return best
+
+
+def _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epochs,
+                save_dir, model_name, save, monitor, track_accuracy, prepare_batch, run):
     best = math.inf if monitor == "loss" else -math.inf
 
     for epoch in range(1, epochs + 1):
+        start = time.perf_counter()
         train = _run_epoch(model, loss_fn, train_loader, device, prepare_batch, track_accuracy,
                            optimizer=optimizer, desc=f"Train {epoch}")
         val = _run_epoch(model, loss_fn, val_loader, device, prepare_batch, track_accuracy,
@@ -78,6 +106,11 @@ def fit(
             best = val[monitor]
             if save:
                 save_checkpoint(model, save_dir, model_name)
+            if run:
+                run.save_weights(model)
+        if run:
+            run.log_epoch(epoch, train, val, lr=optimizer.param_groups[0]["lr"],
+                          seconds=time.perf_counter() - start, improved=improved)
 
         # Loss is the summed batch loss per sample, shown x1000.
         parts = [f"epoch: {epoch}"]
diff --git a/api/src/models/yolo.py b/api/src/models/yolo.py
index 0ddfcbd..b520227 100644
--- a/api/src/models/yolo.py
+++ b/api/src/models/yolo.py
@@ -14,20 +14,20 @@ class SkipBlock(nn.Module):
         super().__init__()
         self.conv = nn.Sequential(
             nn.Conv2d(c_in, c_out, kernel_size, padding=kernel_size // 2),
-            nn.GroupNorm(num_groups=c_out // 8, num_channels=c_out),
+            nn.BatchNorm2d(c_out),
             nn.LeakyReLU(inplace=True),
 
             nn.Conv2d(c_out, c_out, kernel_size, padding=kernel_size // 2),
-            nn.GroupNorm(num_groups=c_out // 8, num_channels=c_out),
+            nn.BatchNorm2d(c_out),
             nn.LeakyReLU(inplace=True),
 
             nn.Conv2d(c_out, c_out, kernel_size, padding=kernel_size // 2),
-            nn.GroupNorm(num_groups=c_out // 8, num_channels=c_out),
+            nn.BatchNorm2d(c_out),
             nn.LeakyReLU(inplace=True)
         )
         self.conv_skip = nn.Sequential(
             nn.Conv2d(c_in, c_out, 1),
-            nn.GroupNorm(num_groups=c_out // 8, num_channels=c_out),
+            nn.BatchNorm2d(c_out),
             nn.LeakyReLU(inplace=True)
         )
 
@@ -54,7 +54,7 @@ class Yolo_model(nn.Module):
 
         self.model = nn.Sequential(
             nn.Conv2d(c_in, c_hidden, kernel_size=3, padding=1),
-            nn.GroupNorm(num_groups=c_hidden // 8, num_channels=c_hidden),
+            nn.BatchNorm2d(c_hidden),
             nn.LeakyReLU(inplace=True),
 
             SkipBlock(c_in=c_hidden, c_out=c_hidden),
@@ -73,7 +73,7 @@ class Yolo_model(nn.Module):
             SkipBlock(c_in=c_hidden * 4, c_out=c_hidden * 4),
 
             nn.Conv2d(c_hidden * 4, c_hidden * 8, kernel_size=3, padding=1),
-            nn.GroupNorm(num_groups=c_hidden // 2, num_channels=c_hidden * 8),
+            nn.BatchNorm2d(c_hidden * 8),
             nn.LeakyReLU(inplace=True),
             nn.Dropout(0.3),
 
diff --git a/api/src/r_cnn/config.py b/api/src/r_cnn/config.py
index e50a65c..58594f7 100644
--- a/api/src/r_cnn/config.py
+++ b/api/src/r_cnn/config.py
@@ -40,6 +40,10 @@ class Config:
     football_model_name: str = "object_detection"
     save: bool = True
 
+    # Run log (src/common/run_log.py): one folder per training run
+    runs_dir: str = "runs"            # empty = no run log
+    run_name: str = ""                # added to the run folder name, e.g. "bn-lr3e-4"
+
     # detect tasks: proposals smaller than min_size or larger than max_size are ignored
     image: str = ""                       # detect-faces: image to run on
     min_size: int = 5
diff --git a/api/src/r_cnn/tasks.py b/api/src/r_cnn/tasks.py
index 0a6d039..3ef39a3 100644
--- a/api/src/r_cnn/tasks.py
+++ b/api/src/r_cnn/tasks.py
@@ -71,7 +71,7 @@ def train_faces(cfg: Config):
 
     fit(model, nn.CrossEntropyLoss(weight=weights), optimizer, train_loader, val_loader, device,
         epochs=cfg.epochs, save_dir=cfg.save_dir, model_name=cfg.faces_model_name, save=cfg.save,
-        monitor="loss", track_accuracy=True)
+        monitor="loss", track_accuracy=True, config=cfg)
 
 
 def detect_faces(cfg: Config):
@@ -103,7 +103,7 @@ def train_football(cfg: Config):
 
     fit(model, nn.CrossEntropyLoss(), optimizer, train_loader, val_loader, device,
         epochs=cfg.epochs, save_dir=cfg.save_dir, model_name=cfg.football_model_name, save=cfg.save,
-        monitor="loss", track_accuracy=True, prepare_batch=prepare_batch)
+        monitor="loss", track_accuracy=True, prepare_batch=prepare_batch, config=cfg)
 
 
 def detect_football(cfg: Config):
diff --git a/api/src/yolo/config.py b/api/src/yolo/config.py
index a701b40..64e0018 100644
--- a/api/src/yolo/config.py
+++ b/api/src/yolo/config.py
@@ -35,6 +35,10 @@ class Config:
     model_name: str = "face_detection_yolo"
     save: bool = True
 
+    # Run log (src/common/run_log.py): one folder per training run
+    runs_dir: str = "runs"            # empty = no run log
+    run_name: str = ""                # added to the run folder name, e.g. "bn-lr3e-4"
+
     # sample / webcam tasks
     checkpoint: str = ""              # empty = //
     threshold: float = 0.95           # confidence needed to draw a box
diff --git a/api/src/yolo/tasks.py b/api/src/yolo/tasks.py
index 6e80fbc..b01c1ca 100644
--- a/api/src/yolo/tasks.py
+++ b/api/src/yolo/tasks.py
@@ -62,7 +62,7 @@ def train(cfg: Config):
 
     fit(model, YoloLoss(), optimizer, train_loader, val_loader, device,
         epochs=cfg.epochs, save_dir=cfg.save_dir, model_name=cfg.model_name, save=cfg.save,
-        monitor="loss")
+        monitor="loss", config=cfg)
 
 
 def view_data(cfg: Config):