removed cloud storage

api/README.md

@@ -92,7 +92,7 @@ api/
 │   │   ├── run_log.py          records every training run to runs/
 │   │   └── visualize.py        matplotlib helpers (lazy import)
 │   ├── bird_cnn/               "bird" task group:   config.py, tasks.py
-│   ├── yolo/                   "yolo" task group:   config.py, tasks.py, dataset.py, loss.py, metrics.py, wider.py
+│   ├── yolo/                   "yolo" task group:   config.py, tasks.py, dataset.py, loss.py, metrics.py, wider.py, merge.py
 │   ├── r_cnn/                  "rcnn" task group:   config.py, tasks.py, dataset.py, crops.py, detect.py
 │   └── production/             "server" task group + the FastAPI app
 │       ├── app.py              FastAPI app, CORS, startup (load models, create DB)
@@ -320,6 +320,8 @@ Code: `src/yolo/` · Defaults: `src/yolo/config.py` · Model: [`Yolo_model`](#yo
 | `--wider-dir` | `""` | convert-wider | WIDER FACE folder with `WIDER_train/`, `WIDER_val/` and `wider_face_split/`. |
 | `--wider-out-dir` | `data/wider_face` | convert-wider | Output folder; gets `train/` and `test/`. |
 | `--wider-min-face` | `2.0` | convert-wider | Drop faces smaller than this (px, after resizing to `--image-size`). |
+| `--merge-dirs` | `""` | merge-coco | COCO folders to combine, comma separated. |
+| `--merge-out-dir` | `""` | merge-coco | Output folder for the combined dataset. Must not exist yet. |
 | `--c-hidden` | `16` | all | Base channel count (c, 2c, 4c, 8c). |
 | `--boxes` | `1` | all | Boxes per cell. Only 1 is supported by the dataset, loss and decoder. |
 | `--labels` | `1` | all | Number of classes. Must equal the number of categories in the COCO file. |
@@ -336,6 +338,7 @@ Code: `src/yolo/` · Defaults: `src/yolo/config.py` · Model: [`Yolo_model`](#yo
 | `--save-dir` | `saved_models` | train, sample, webcam | Root folder of checkpoints. |
 | `--model-name` | `face_detection_yolo` | train, sample, webcam | Checkpoint name. |
 | `--save` / `--no-save` | `--save` | train | Save the best checkpoint. |
+| `--save-every-epoch` / `--no-save-every-epoch` | off | train | With `--save`: also keep every epoch as `<model-name>_epochNNN` next to the best checkpoint. |
 | `--runs-dir` | `runs` | train | Run log folder. Empty disables it. |
 | `--run-name` | `""` | train | Label appended to the run folder name. |
 | `--checkpoint` | `""` | sample, webcam | Explicit weights file. Empty = `<save-dir>/<model-name>/<model-name>`. |
@@ -461,6 +464,33 @@ python -m src.main yolo.convert-wider --wider-dir ~/Downloads/wider_face --image
 python -m src.main yolo.train --train-dir data/wider_face/train --val-dir data/wider_face/test
 ```
 
+
+#### `yolo.merge-coco`
+
+Combines two or more COCO folders into one, for example to train on WIDER and
+`faces_scenery` together. Run it once per split.
+
+```bash
+python -m src.main yolo.merge-coco \
+  --merge-dirs data/wider_faces_coco/train,data/faces_scenery/train --merge-out-dir data/faces_combined/train
+python -m src.main yolo.merge-coco \
+  --merge-dirs data/wider_faces_coco/test,data/faces_scenery/test --merge-out-dir data/faces_combined/test
+```
+
+- **Images:** hard-linked into the output folder, so they take no extra disk
+  space. If the folders are on different drives, they are copied instead.
+  Each file is renamed `<source>__<file_name>`, e.g.
+  `faces_scenery_train__0001.jpg`, so names from different sources never clash.
+- **Ids:** image and annotation ids are renumbered.
+- **Categories:** merged by name and numbered 1, 2, … in order of first
+  appearance. The same class in both sources becomes one class. The task
+  prints the merged list; `--labels` must equal its length.
+- **Checks:** before writing anything, it stops if a source has no annotation
+  file, if an image listed in it is missing, or if `--merge-out-dir` already exists.
+
+The sources are combined as they are. Prepare each one with the same
+`--image-size` and `--grid` you train with (see `yolo.convert-wider`).
+
 ---
 
 ### `rcnn.*`: R-CNN-style experiments
@@ -890,6 +920,8 @@ far slower than the single-shot YOLO model. That is why the website uses YOLO.
    - any other metric, e.g. `"acc"` (bird) or `"conf_ap"` (YOLO): higher is better.
 4. **Save on improvement:** if it improved, saves the checkpoint (`--save`)
    and the run's `best.pt`.
+   With `save_every_epoch` (YOLO: `--save-every-epoch`), every epoch is also
+   saved as `<model_name>_epochNNN`, improved or not.
 5. **Log:** prints one summary line and appends a row to the run log.
 6. **Schedule:** steps the optional `scheduler` (YOLO uses `warmup_cosine`).
 
@@ -909,6 +941,11 @@ DataLoader passes for different worker counts.
 - **Save path:** `<save_dir>/<model_name>/<model_name>`, for example
   `saved_models/face_detection_yolo/face_detection_yolo`. Only the **best**
   epoch is kept, and each improvement overwrites the file.
+- **Every epoch (YOLO, `--save-every-epoch`):** each epoch is additionally
+  saved next to it as `<model_name>_epochNNN`, e.g.
+  `saved_models/face_detection_yolo/face_detection_yolo_epoch042`. Load one with
+  `--checkpoint`. Each file is the model's full size, so 200 epochs of c32
+  take about 2.7 GB.
 - **Format:** a plain `state_dict` (`torch.save(model.state_dict())`).
 - **Loading:** `torch.load(..., weights_only=True, map_location=device)`, then
   `eval()`. Weights trained on a GPU therefore load on a CPU-only machine.
@@ -1244,6 +1281,7 @@ folder. Unpack it in `api/` with `7z x data.7z`.
 | `faces/train/` | COCO | `rcnn.create-dataset` | Face set (3,910 images). |
 | `faces_processed/face/`, `background/` | ImageFolder | `rcnn.train-faces` | Crops made by `rcnn.create-dataset`. |
 | `wider_face/train/`, `test/` | COCO | `yolo.*` | Created by `yolo.convert-wider` from WIDER FACE, category 1 = face. Not in `data.7z`. |
+| any `merge-coco` output | COCO | `yolo.*` | Created by `yolo.merge-coco` from other COCO folders. Not in `data.7z`. |
 | `football/train/`, `valid/` | COCO | `rcnn.*-football` | Roboflow football players: 1 = ball, 2 = goalkeeper, 3 = player, 4 = referee. |
 | `faces_2/`, `mnist/`, WIDER Face archives | various | none | Not used by the current code. |
 

api/src/common/checkpoints.py

@@ -3,13 +3,16 @@ from pathlib import Path
 import torch
 
 
-def checkpoint_path(save_dir: str | Path, model_name: str) -> Path:
-    # Layout kept from the original code: <save_dir>/<name>/<name>
-    return Path(save_dir) / model_name / model_name
+def checkpoint_path(save_dir: str | Path, model_name: str, epoch: int | None = None) -> Path:
+    # Layout kept from the original code: <save_dir>/<name>/<name>,
+    # per-epoch checkpoints next to it as <name>_epoch007
+    file_name = model_name if epoch is None else f"{model_name}_epoch{epoch:03d}"
+    return Path(save_dir) / model_name / file_name
 
 
-def save_checkpoint(model: torch.nn.Module, save_dir: str | Path, model_name: str) -> Path:
-    path = checkpoint_path(save_dir, model_name)
+def save_checkpoint(model: torch.nn.Module, save_dir: str | Path, model_name: str,
+                    epoch: int | None = None) -> Path:
+    path = checkpoint_path(save_dir, model_name, epoch)
     path.parent.mkdir(parents=True, exist_ok=True)
     torch.save(model.state_dict(), path)
     return path

api/src/common/training.py

@@ -112,6 +112,7 @@ def fit(
     val_metrics=None,               # f(preds, targets) -> dict of extra metrics over the whole val set
     config=None,                    # the task's Config; enables the run log
     scheduler=None,                 # LR scheduler, stepped once per epoch
+    save_every_epoch: bool = False, # with save: also keep every epoch as <model_name>_epochNNN
 ):
     run = None
     if config is not None and config.runs_dir:
@@ -120,7 +121,7 @@ def fit(
     try:
         best = _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epochs,
                            save_dir, model_name, save, monitor, track_accuracy, prepare_batch, val_metrics, run,
-                           scheduler)
+                           scheduler, save_every_epoch)
     except KeyboardInterrupt:
         if run:
             run.finish("interrupted")
@@ -136,7 +137,7 @@ def fit(
 
 def _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epochs,
                 save_dir, model_name, save, monitor, track_accuracy, prepare_batch, val_metrics, run,
-                scheduler):
+                scheduler, save_every_epoch):
     best = math.inf if monitor == "loss" else -math.inf
 
     for epoch in range(1, epochs + 1):
@@ -153,6 +154,8 @@ def _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epo
                 save_checkpoint(model, save_dir, model_name)
             if run:
                 run.save_weights(model)
+        if save and save_every_epoch:
+            save_checkpoint(model, save_dir, model_name, epoch=epoch)
         if run:
             run.log_epoch(epoch, train, val, lr=optimizer.param_groups[0]["lr"],
                           seconds=time.perf_counter() - start, improved=improved)
@@ -166,6 +169,8 @@ def _fit_epochs(model, loss_fn, optimizer, train_loader, val_loader, device, epo
         parts += [f"val {k}: {v:.3f}" for k, v in val.items() if k not in ("loss", "acc")]
         if improved and save:
             parts.append("saved")
+        if save and save_every_epoch:
+            parts.append(f"kept epoch {epoch}")
         print(" | ".join(parts))
 
     return best

api/src/models/yolo.py

@@ -51,8 +51,7 @@ class Yolo_model(nn.Module):
         self.size_activation = size_activation
 
         self.model = nn.Sequential(
-            #nn.Conv2d(c_in, c_hidden, kernel_size=3, padding=1, stride=2),
-            nn.Conv2d(c_in, c_hidden, kernel_size=3, padding=1),
+            nn.Conv2d(c_in, c_hidden, kernel_size=3, padding=1, stride=2),
             nn.BatchNorm2d(c_hidden),
             nn.LeakyReLU(inplace=True),
 
@@ -61,8 +60,7 @@ class Yolo_model(nn.Module):
             SkipBlock(c_in=c_hidden, c_out=c_hidden),
             SkipBlock(c_in=c_hidden, c_out=c_hidden),
 
-            #nn.Conv2d(c_hidden, c_hidden * 2, kernel_size=5, padding=2, stride=4),
-            nn.Conv2d(c_hidden, c_hidden * 2, kernel_size=3, padding=1, stride=2),
+            nn.Conv2d(c_hidden, c_hidden * 2, kernel_size=5, padding=2, stride=4),
             nn.LeakyReLU(inplace=True),
 
             SkipBlock(c_in=c_hidden * 2, c_out=c_hidden * 2),
@@ -85,7 +83,6 @@ class Yolo_model(nn.Module):
             nn.BatchNorm2d(c_hidden * 8),
             nn.LeakyReLU(inplace=True),
 
-            nn.AdaptiveAvgPool2d((grid, grid)),
             nn.Conv2d(c_hidden * 8, boxes * 5 + labels, kernel_size=1)
         )
 

api/src/production/settings.py

@@ -24,12 +24,12 @@ BIRD_MODEL_FILE = "bird_cnn"
 BIRD_IMAGE_SIZE = 64
 BIRD_C_HIDDEN = 16
 
-FACE_MODEL_FILE = "face_detection_yolo_2"
-FACE_IMAGE_SIZE = 64
-FACE_GRID = 6
-FACE_C_HIDDEN = 8
+FACE_MODEL_FILE = "face_detection_yolo_3"
+FACE_IMAGE_SIZE = 256
+FACE_GRID = 8
+FACE_C_HIDDEN = 16
 FACE_SIZE_ACTIVATION = "sigmoid"   # must match how the deployed weights were trained
-FACE_THRESHOLD = 0.9
+FACE_THRESHOLD = 0.5
 
 # ---- Resource limits (the server is weak, keep everything small and bounded) ----
 # Number of predictions (bird or face) that may run at the same time. (env)

api/src/yolo/config.py

@@ -20,6 +20,10 @@ class Config:
     wider_out_dir: str = "data/wider_face"
     wider_min_face: float = 2.0       # drop smaller faces (px after resizing to image_size)
 
+    # merge-coco: combine COCO folders into one
+    merge_dirs: str = ""              # comma separated, e.g. data/wider_faces_coco/train,data/faces_scenery/train
+    merge_out_dir: str = ""
+
     # Model
     c_hidden: int = 16
     boxes: int = 1
@@ -41,6 +45,7 @@ class Config:
     save_dir: str = "saved_models"
     model_name: str = "face_detection_yolo"
     save: bool = True
+    save_every_epoch: bool = False    # with --save: also keep every epoch as <model_name>_epochNNN
 
     # Run log (src/common/run_log.py): one folder per training run
     runs_dir: str = "runs"            # empty = no run log

api/src/yolo/tasks.py

@@ -17,6 +17,7 @@ from src.yolo.config import Config
 from src.yolo.dataset import YoloDataset
 from src.yolo.loss import YoloLoss
 from src.yolo.metrics import detection_metrics
+from src.yolo.merge import merge_coco
 from src.yolo.wider import convert_wider
 
 
@@ -65,6 +66,7 @@ def train(cfg: Config):
 
     fit(model, YoloLoss(), optimizer, train_loader, val_loader, device,
         epochs=cfg.epochs, save_dir=cfg.save_dir, model_name=cfg.model_name, save=cfg.save,
+        save_every_epoch=cfg.save_every_epoch,
         monitor="conf_ap", val_metrics=detection_metrics, config=cfg, scheduler=scheduler)
 
 
@@ -150,4 +152,5 @@ TASKS = {
     "benchmark-workers": benchmark_workers,
     "webcam": webcam,
     "convert-wider": convert_wider,
+    "merge-coco": merge_coco,
 }

api/train_test.sh

@@ -2,8 +2,8 @@
 
 
 for ((c = 1; c <= 32; c *= 2)); do
-    python -m src.main yolo.train --train-dir ./data/wider_faces_coco/train/ --val-dir ./data/wider_faces_coco/test/ \
-        --no-augment-val --c-hidden "$c" --batch-size 64 --epochs 200 --run-name "bn-wider-faces-c$c" \
+    python -m src.main yolo.train --train-dir ./data/faces_combined/train/ --val-dir ./data/faces_combined/test/ \
+        --no-augment-val --c-hidden "$c" --batch-size 64 --epochs 200 --run-name "bn-combined-faces-c$c" \
         --no-save --train-workers 4 --image-size 256 --grid 8 \
         --learning-rate 1e-3 --weight-decay 0.05 --warmup-epochs 3 --min-learning-rate 1e-5
 done

caddy/Caddyfile

@@ -13,10 +13,6 @@
 	log
 }
 
-www.marvinkrausser.com {
-	redir https://marvinkrausser.com{uri} permanent
-}
-
 marvinkrausser.com {
 	import common
 	reverse_proxy cnn_website:8080
@@ -60,16 +56,21 @@ game.marvinkrausser.com {
 		reverse_proxy wizard_website:8082
 	}
 
+	# handle_path takes a single path, so use named matchers to also match the
+	# bare prefix. The prefixes are stripped, so the backends see "/..." only.
+	@actionblocks_website path /actionblocks/website /actionblocks/website/*
+	handle @actionblocks_website {
+		uri strip_prefix /actionblocks/website
+		reverse_proxy actionblocks-website:80
+	}
+
+	@actionblocks_server path /actionblocks/server /actionblocks/server/*
+	handle @actionblocks_server {
+		uri strip_prefix /actionblocks/server
+		reverse_proxy actionblocks-server:8080
+	}
+
 	handle {
 		respond 404
 	}
 }
-
-cloud.marvinkrausser.com {
-    reverse_proxy nextcloud-nextcloud-1:80
-
-    redir /.well-known/carddav /remote.php/dav 301
-    redir /.well-known/caldav /remote.php/dav 301
-
-    header Strict-Transport-Security "max-age=15552000; includeSubDomains"
-}

compose.yaml

@@ -23,7 +23,8 @@ services:
       - portfolio_network
       - wizard_website
       - wizard_api
-      - cloud_network
+      - actionBlocks_website
+      - actionBlocks_server
     pull_policy: never
     container_name: caddy_proxy
 
@@ -78,8 +79,7 @@ networks:
     name: "wizard_website"
   wizard_api:
     name: "wizard_api"
-  cloud_network:
-    name: cloud_network
-    ipam:
-      config:
-        - subnet: 172.30.84.0/24
+  actionBlocks_website:
+    name: "actionBlocks_website"
+  actionBlocks_server:
+    name: "actionBlocks_server"