feat(monitoring): add update-run log, fix scheduler skipping stale customer containers

The automatic scheduler only recreated customer containers when it had just
found a NEW image on Docker Hub. If an image was already pulled locally
(e.g. from an earlier manual pull, or auto-apply was off at the time) but
customer containers hadn't been recreated yet, the daily check reported
"already up to date" and silently never touched those customers again —
the recreate step needs to run independently of whether Hub had anything
new, based on each customer's actual container-vs-local-image state.

Also adds a persistent UpdateRunLog: every scheduled and manual pull/update
run now records per-image pull results and per-customer recreate results,
visible in Monitoring > Update Log, instead of only existing in container
logs.

app/database.py

@@ -45,6 +45,7 @@ def init_db() -> None:
         Deployment,
         DeploymentLog,
         SystemConfig,
+        UpdateRunLog,
         User,
     )
 

app/models.py

@@ -1,5 +1,6 @@
 """SQLAlchemy ORM models for NetBird MSP Appliance."""
 
+import json
 from datetime import datetime
 from typing import Optional
 
@@ -321,6 +322,56 @@ class DeploymentLog(Base):
         }
 
 
+class UpdateRunLog(Base):
+    """Audit trail for NetBird Docker image update runs (pull + container recreate).
+
+    Written for both scheduled (automatic) and manual (button-triggered)
+    runs, so the actual outcome — which image pulls failed, which customer
+    container recreates failed — is visible in the UI instead of only in
+    container logs.
+    """
+
+    __tablename__ = "update_run_logs"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    run_type: Mapped[str] = mapped_column(String(20), nullable=False)  # "scheduled" | "manual"
+    trigger: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)  # username, or "scheduler"
+    started_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
+    finished_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
+    status: Mapped[str] = mapped_column(
+        String(20), default="running"
+    )  # running | no_update | success | partial | failed
+    any_update_available: Mapped[Optional[bool]] = mapped_column(Boolean, nullable=True)
+    pull_attempted: Mapped[bool] = mapped_column(Boolean, default=False)
+    pull_success: Mapped[Optional[bool]] = mapped_column(Boolean, nullable=True)
+    pull_details: Mapped[Optional[str]] = mapped_column(Text, nullable=True)  # JSON: {image: {success, error}}
+    apply_attempted: Mapped[bool] = mapped_column(Boolean, default=False)
+    customers_updated: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
+    customers_total: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
+    customer_details: Mapped[Optional[str]] = mapped_column(Text, nullable=True)  # JSON list
+    error: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+
+    def to_dict(self) -> dict:
+        """Serialize to dict, parsing the embedded JSON blobs."""
+        return {
+            "id": self.id,
+            "run_type": self.run_type,
+            "trigger": self.trigger,
+            "started_at": self.started_at.isoformat() if self.started_at else None,
+            "finished_at": self.finished_at.isoformat() if self.finished_at else None,
+            "status": self.status,
+            "any_update_available": self.any_update_available,
+            "pull_attempted": self.pull_attempted,
+            "pull_success": self.pull_success,
+            "pull_details": json.loads(self.pull_details) if self.pull_details else None,
+            "apply_attempted": self.apply_attempted,
+            "customers_updated": self.customers_updated,
+            "customers_total": self.customers_total,
+            "customer_details": json.loads(self.customer_details) if self.customer_details else None,
+            "error": self.error,
+        }
+
+
 class User(Base):
     """Admin user model."""
 

app/routers/deployments.py

@@ -9,7 +9,7 @@ from sqlalchemy.orm import Session
 from app.database import SessionLocal, get_db
 from app.dependencies import get_current_user
 from app.models import Customer, Deployment, SystemConfig, User
-from app.services import docker_service, image_service, netbird_client_update_service, netbird_service
+from app.services import docker_service, image_service, netbird_client_update_service, netbird_service, update_log_service
 from app.utils.security import decrypt_value, encrypt_value
 from app.utils.validators import NetbirdApiTokenPayload, NetbirdClientAutoUpdatePayload
 
@@ -246,9 +246,18 @@ async def update_customer_images(
             status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="System not configured."
         )
 
+    started_at = datetime.utcnow()
     instance_dir = f"{config.data_dir}/{customer.subdomain}"
     result = await image_service.update_customer_containers(instance_dir, deployment.container_prefix)
 
+    update_log_service.record_run(
+        db, run_type="manual", trigger=current_user.username, started_at=started_at,
+        apply_attempted=True,
+        customer_results=[{
+            "customer_name": customer.name, "success": result["success"], "error": result.get("error"),
+        }],
+    )
+
     if not result["success"]:
         raise HTTPException(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,

app/routers/monitoring.py

@@ -4,6 +4,7 @@ import asyncio
 import logging
 import platform
 import time
+from datetime import datetime
 from typing import Any
 
 import psutil
@@ -12,8 +13,8 @@ from sqlalchemy.orm import Session
 
 from app.database import SessionLocal, get_db
 from app.dependencies import get_current_user
-from app.models import Customer, Deployment, SystemConfig, User
-from app.services import docker_service, image_service, netbird_client_update_service
+from app.models import Customer, Deployment, SystemConfig, UpdateRunLog, User
+from app.services import docker_service, image_service, netbird_client_update_service, update_log_service
 from app.utils.security import decrypt_value
 from app.utils.validators import NetbirdClientAutoUpdatePayload
 
@@ -194,14 +195,25 @@ async def pull_all_netbird_images(
         config.netbird_dashboard_image,
     ]
 
+    username = current_user.username
+
     async def _pull_bg() -> None:
+        started_at = datetime.utcnow()
         bg_db = SessionLocal()
         try:
             cfg = bg_db.query(SystemConfig).filter(SystemConfig.id == 1).first()
-            if cfg:
-                await image_service.pull_all_images(cfg)
-        except Exception:
+            if not cfg:
+                return
+            pull_result = await image_service.pull_all_images(cfg)
+            update_log_service.record_run(
+                bg_db, run_type="manual", trigger=username, started_at=started_at,
+                any_update_available=True, pull_attempted=True, pull_results=pull_result["results"],
+            )
+        except Exception as exc:
             logger.exception("Background image pull failed")
+            update_log_service.record_run(
+                bg_db, run_type="manual", trigger=username, started_at=started_at, error=str(exc),
+            )
         finally:
             bg_db.close()
 
@@ -285,6 +297,28 @@ async def apply_netbird_client_updates_to_all(
     }
 
 
+@router.get("/update-logs")
+async def get_update_logs(
+    limit: int = 30,
+    current_user: User = Depends(get_current_user),
+    db: Session = Depends(get_db),
+) -> list[dict[str, Any]]:
+    """Return the most recent NetBird image update runs (scheduled and manual).
+
+    Shows exactly what happened on each run: whether an image pull was
+    attempted and its per-image result, and whether customer containers
+    were recreated and which ones failed — so this doesn't have to be
+    reconstructed from container logs.
+    """
+    logs = (
+        db.query(UpdateRunLog)
+        .order_by(UpdateRunLog.started_at.desc())
+        .limit(min(limit, 100))
+        .all()
+    )
+    return [log.to_dict() for log in logs]
+
+
 @router.post("/customers/update-all")
 async def update_all_customers(
     current_user: User = Depends(get_current_user),
@@ -299,6 +333,7 @@ async def update_all_customers(
     if current_user.role != "admin":
         raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin only.")
 
+    started_at = datetime.utcnow()
     config = db.query(SystemConfig).filter(SystemConfig.id == 1).first()
     if not config:
         raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="System not configured.")
@@ -317,6 +352,10 @@ async def update_all_customers(
             })
 
     if not to_update:
+        update_log_service.record_run(
+            db, run_type="manual", trigger=current_user.username,
+            started_at=started_at, apply_attempted=True, customer_results=[],
+        )
         return {"message": "All customers are already up to date.", "updated": 0, "results": []}
 
     # Update customers sequentially — one at a time. A failure for one
@@ -343,6 +382,10 @@ async def update_all_customers(
         })
 
     success_count = sum(1 for r in update_results if r["success"])
+    update_log_service.record_run(
+        db, run_type="manual", trigger=current_user.username,
+        started_at=started_at, apply_attempted=True, customer_results=update_results,
+    )
     return {
         "message": f"Updated {success_count} of {len(update_results)} customer(s).",
         "updated": success_count,

app/services/scheduler_service.py

@@ -12,7 +12,7 @@ from datetime import datetime, timedelta
 
 from app.database import SessionLocal
 from app.models import Deployment, SystemConfig
-from app.services import image_service, netbird_client_update_service
+from app.services import image_service, netbird_client_update_service, update_log_service
 from app.utils.security import decrypt_value, encrypt_value
 
 logger = logging.getLogger(__name__)
@@ -125,29 +125,49 @@ async def _tick() -> None:
         logger.info(
             "Running scheduled NetBird image update check (auto-apply=%s)...", apply_enabled
         )
-        await _run_check_and_optionally_apply(config, apply_enabled)
+        await _run_check_and_optionally_apply(config, apply_enabled, started_at=now)
     finally:
         db.close()
 
 
-async def _run_check_and_optionally_apply(config: SystemConfig, apply_enabled: bool) -> None:
-    hub_status = await image_service.check_all_images(config)
-    if not hub_status["any_update_available"]:
-        logger.info("Scheduled check: all NetBird images already up to date.")
-        return
+async def _run_check_and_optionally_apply(config: SystemConfig, apply_enabled: bool, started_at: datetime) -> None:
+    """Check Docker Hub, pull if needed, then (if enabled) recreate any
+    customer container that isn't running the locally-pulled image yet.
 
-    logger.info("Scheduled check: new NetBird image(s) available — pulling.")
-    pull_result = await image_service.pull_all_images(config)
-    if not pull_result["all_success"]:
-        logger.error("Scheduled image pull had failures: %s", pull_result["results"])
-
-    if not apply_enabled:
-        logger.info("Auto-apply disabled — images pulled, customer containers left untouched.")
-        return
-
-    db = SessionLocal()
+    The "recreate" step is intentionally NOT gated on a Hub update having
+    just been found: a customer can already be behind the *locally pulled*
+    image from an earlier check/pull that was never applied (e.g. auto-apply
+    was off at the time, or a previous run failed) — that backlog must keep
+    being cleared on every run, not just on runs that also found something
+    new on Hub, otherwise it silently never happens.
+    """
+    log_db = SessionLocal()
     try:
-        deployments = db.query(Deployment).all()
+        hub_status = await image_service.check_all_images(config)
+        any_update = hub_status["any_update_available"]
+
+        pull_attempted = False
+        pull_results = None
+        if any_update:
+            logger.info("Scheduled check: new NetBird image(s) available — pulling.")
+            pull_attempted = True
+            pull_result = await image_service.pull_all_images(config)
+            pull_results = pull_result["results"]
+            if not pull_result["all_success"]:
+                logger.error("Scheduled image pull had failures: %s", pull_results)
+        else:
+            logger.info("Scheduled check: all NetBird images already up to date on Docker Hub.")
+
+        if not apply_enabled:
+            logger.info("Auto-apply disabled — customer containers left untouched.")
+            update_log_service.record_run(
+                log_db, run_type="scheduled", trigger="scheduler",
+                started_at=started_at, any_update_available=any_update,
+                pull_attempted=pull_attempted, pull_results=pull_results,
+            )
+            return
+
+        deployments = log_db.query(Deployment).all()
         to_update = []
         for dep in deployments:
             cs = image_service.get_customer_container_image_status(dep.container_prefix, config)
@@ -159,16 +179,32 @@ async def _run_check_and_optionally_apply(config: SystemConfig, apply_enabled: b
                     "customer_name": customer.name,
                 })
         logger.info("Scheduled auto-apply: updating %d customer(s)...", len(to_update))
+        customer_results = []
         for entry in to_update:
             try:
                 res = await image_service.update_customer_containers(
                     entry["instance_dir"], entry["project_name"]
                 )
-                logger.info(
-                    "Scheduled update for %s: %s",
-                    entry["customer_name"], "OK" if res["success"] else res.get("error"),
-                )
-            except Exception:
+                ok = res["success"]
+                err = res.get("error")
+            except Exception as exc:
                 logger.exception("Scheduled update failed for %s", entry["customer_name"])
+                ok = False
+                err = str(exc)
+            logger.info("Scheduled update for %s: %s", entry["customer_name"], "OK" if ok else err)
+            customer_results.append({"customer_name": entry["customer_name"], "success": ok, "error": err})
+
+        update_log_service.record_run(
+            log_db, run_type="scheduled", trigger="scheduler",
+            started_at=started_at, any_update_available=any_update,
+            pull_attempted=pull_attempted, pull_results=pull_results,
+            apply_attempted=True, customer_results=customer_results,
+        )
+    except Exception as exc:
+        logger.exception("Scheduled update run failed")
+        update_log_service.record_run(
+            log_db, run_type="scheduled", trigger="scheduler",
+            started_at=started_at, error=str(exc),
+        )
     finally:
-        db.close()
+        log_db.close()

app/services/update_log_service.py

@@ -0,0 +1,84 @@
+"""Persists the outcome of every NetBird Docker image update run.
+
+Both the automatic scheduler and the manual "Check/Pull/Update" buttons in
+the UI write here, so it's visible afterwards which run pulled which image
+successfully, and which customer container recreate failed — instead of
+that information only existing in container logs.
+"""
+
+import json
+import logging
+from datetime import datetime
+from typing import Any, Optional
+
+from sqlalchemy.orm import Session
+
+from app.models import UpdateRunLog
+
+logger = logging.getLogger(__name__)
+
+
+def record_run(
+    db: Session,
+    *,
+    run_type: str,
+    trigger: Optional[str],
+    started_at: datetime,
+    any_update_available: Optional[bool] = None,
+    pull_attempted: bool = False,
+    pull_results: Optional[dict[str, Any]] = None,
+    apply_attempted: bool = False,
+    customer_results: Optional[list[dict[str, Any]]] = None,
+    error: Optional[str] = None,
+) -> UpdateRunLog:
+    """Write one completed update-run entry.
+
+    Args:
+        run_type: "scheduled" or "manual".
+        trigger: username for manual runs, "scheduler" for automatic ones.
+        pull_results: {image: {"success": bool, "error": str|None}} as returned
+            by image_service.pull_all_images()["results"].
+        customer_results: list of {"customer_name", "success", "error"} as
+            returned by the update-all / update-customer flows.
+    """
+    pull_success = None
+    if pull_results:
+        pull_success = all(r.get("success") for r in pull_results.values())
+
+    customers_total = len(customer_results) if customer_results is not None else None
+    customers_updated = (
+        sum(1 for r in customer_results if r.get("success")) if customer_results is not None else None
+    )
+
+    if error:
+        status = "failed"
+    elif pull_attempted and pull_success is False:
+        status = "failed"
+    elif apply_attempted and customers_total and customers_updated != customers_total:
+        status = "partial"
+    elif not pull_attempted and not customers_total:
+        # Nothing to pull (Hub already matched local) and no customer
+        # container was behind the locally pulled image either.
+        status = "no_update"
+    else:
+        status = "success"
+
+    log = UpdateRunLog(
+        run_type=run_type,
+        trigger=trigger,
+        started_at=started_at,
+        finished_at=datetime.utcnow(),
+        status=status,
+        any_update_available=any_update_available,
+        pull_attempted=pull_attempted,
+        pull_success=pull_success,
+        pull_details=json.dumps(pull_results) if pull_results is not None else None,
+        apply_attempted=apply_attempted,
+        customers_updated=customers_updated,
+        customers_total=customers_total,
+        customer_details=json.dumps(customer_results) if customer_results is not None else None,
+        error=error,
+    )
+    db.add(log)
+    db.commit()
+    return log