diff --git a/api/README.md b/api/README.md index 02ad3d5..bb87b52 100644 --- a/api/README.md +++ b/api/README.md @@ -338,6 +338,7 @@ Code: `src/yolo/` · Defaults: `src/yolo/config.py` · Model: [`Yolo_model`](#yo | `--save-dir` | `saved_models` | train, sample, webcam | Root folder of checkpoints. | | `--model-name` | `face_detection_yolo` | train, sample, webcam | Checkpoint name. | | `--save` / `--no-save` | `--save` | train | Save the best checkpoint. | +| `--save-every-epoch` / `--no-save-every-epoch` | off | train | With `--save`: also keep every epoch as `_epochNNN` next to the best checkpoint. | | `--runs-dir` | `runs` | train | Run log folder. Empty disables it. | | `--run-name` | `""` | train | Label appended to the run folder name. | | `--checkpoint` | `""` | sample, webcam | Explicit weights file. Empty = `//`. | @@ -919,6 +920,8 @@ far slower than the single-shot YOLO model. That is why the website uses YOLO. - any other metric, e.g. `"acc"` (bird) or `"conf_ap"` (YOLO): higher is better. 4. **Save on improvement:** if it improved, saves the checkpoint (`--save`) and the run's `best.pt`. + With `save_every_epoch` (YOLO: `--save-every-epoch`), every epoch is also + saved as `_epochNNN`, improved or not. 5. **Log:** prints one summary line and appends a row to the run log. 6. **Schedule:** steps the optional `scheduler` (YOLO uses `warmup_cosine`). @@ -938,6 +941,11 @@ DataLoader passes for different worker counts. - **Save path:** `//`, for example `saved_models/face_detection_yolo/face_detection_yolo`. Only the **best** epoch is kept, and each improvement overwrites the file. +- **Every epoch (YOLO, `--save-every-epoch`):** each epoch is additionally + saved next to it as `_epochNNN`, e.g. + `saved_models/face_detection_yolo/face_detection_yolo_epoch042`. Load one with + `--checkpoint`. Each file is the model's full size, so 200 epochs of c32 + take about 2.7 GB. - **Format:** a plain `state_dict` (`torch.save(model.state_dict())`). - **Loading:** `torch.load(..., weights_only=True, map_location=device)`, then `eval()`. Weights trained on a GPU therefore load on a CPU-only machine. diff --git a/api/src/common/checkpoints.py b/api/src/common/checkpoints.py index de77aea..4b48adf 100644 --- a/api/src/common/checkpoints.py +++ b/api/src/common/checkpoints.py @@ -3,13 +3,16 @@ from pathlib import Path import torch -def checkpoint_path(save_dir: str | Path, model_name: str) -> Path: - # Layout kept from the original code: // - return Path(save_dir) / model_name / model_name +def checkpoint_path(save_dir: str | Path, model_name: str, epoch: int | None = None) -> Path: + # Layout kept from the original code: //, + # per-epoch checkpoints next to it as _epoch007 + file_name = model_name if epoch is None else f"{model_name}_epoch{epoch:03d}" + return Path(save_dir) / model_name / file_name -def save_checkpoint(model: torch.nn.Module, save_dir: str | Path, model_name: str) -> Path: - path = checkpoint_path(save_dir, model_name) +def save_checkpoint(model: torch.nn.Module, save_dir: str | Path, model_name: str, + epoch: int | None = None) -> Path: + path = checkpoint_path(save_dir, model_name, epoch) path.parent.mkdir(parents=True, exist_ok=True) torch.save(model.state_dict(), path) return path diff --git a/api/src/common/training.py b/api/src/common/training.py index 77c4892..472b622 100644 --- a/api/src/common/training.py +++ b/api/src/common/training.py @@ -112,6 +112,7 @@ def fit( val_metrics=None, # f(preds, targets) -> dict of extra metrics over the whole val set config=None, # the task's Config; enables the run log scheduler=None, # LR scheduler, stepped once per epoch + save_every_epoch: bool = False, # with save: also keep every epoch as _epochNNN ): run = None if config is not None and config.runs_dir: @@ -120,7 +121,7 @@ def fit( try: best = _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epochs, save_dir, model_name, save, monitor, track_accuracy, prepare_batch, val_metrics, run, - scheduler) + scheduler, save_every_epoch) except KeyboardInterrupt: if run: run.finish("interrupted") @@ -136,7 +137,7 @@ def fit( def _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epochs, save_dir, model_name, save, monitor, track_accuracy, prepare_batch, val_metrics, run, - scheduler): + scheduler, save_every_epoch): best = math.inf if monitor == "loss" else -math.inf for epoch in range(1, epochs + 1): @@ -153,6 +154,8 @@ def _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epo save_checkpoint(model, save_dir, model_name) if run: run.save_weights(model) + if save and save_every_epoch: + save_checkpoint(model, save_dir, model_name, epoch=epoch) if run: run.log_epoch(epoch, train, val, lr=optimizer.param_groups[0]["lr"], seconds=time.perf_counter() - start, improved=improved) @@ -166,6 +169,8 @@ def _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epo parts += [f"val {k}: {v:.3f}" for k, v in val.items() if k not in ("loss", "acc")] if improved and save: parts.append("saved") + if save and save_every_epoch: + parts.append(f"kept epoch {epoch}") print(" | ".join(parts)) return best diff --git a/api/src/yolo/config.py b/api/src/yolo/config.py index df06daa..142ea95 100644 --- a/api/src/yolo/config.py +++ b/api/src/yolo/config.py @@ -45,6 +45,7 @@ class Config: save_dir: str = "saved_models" model_name: str = "face_detection_yolo" save: bool = True + save_every_epoch: bool = False # with --save: also keep every epoch as _epochNNN # Run log (src/common/run_log.py): one folder per training run runs_dir: str = "runs" # empty = no run log diff --git a/api/src/yolo/tasks.py b/api/src/yolo/tasks.py index 7ce9647..5894ba8 100644 --- a/api/src/yolo/tasks.py +++ b/api/src/yolo/tasks.py @@ -66,6 +66,7 @@ def train(cfg: Config): fit(model, YoloLoss(), optimizer, train_loader, val_loader, device, epochs=cfg.epochs, save_dir=cfg.save_dir, model_name=cfg.model_name, save=cfg.save, + save_every_epoch=cfg.save_every_epoch, monitor="conf_ap", val_metrics=detection_metrics, config=cfg, scheduler=scheduler) diff --git a/caddy/Caddyfile b/caddy/Caddyfile index 6f6bbad..0ed100d 100644 --- a/caddy/Caddyfile +++ b/caddy/Caddyfile @@ -13,10 +13,6 @@ log } -www.marvinkrausser.com { - redir https://marvinkrausser.com{uri} permanent -} - marvinkrausser.com { import common reverse_proxy cnn_website:8080