fix(monitoring): repair silent-false-positive update badge + auto-update scheduling

Customer container status checks looked up containers by an exact expected name. When a docker compose recreate got interrupted (e.g. a hung command previously killed the whole update-all batch on timeout), Compose could leave the old container renamed with a random hash prefix instead of removed. The exact-name lookup then found nothing, returned None, and that silently counted as "up to date" (green "Aktuell") instead of surfacing as unknown — affecting 5 customers on the appliance whose containers were actually still running under orphaned names. - _run_cmd no longer raises on subprocess timeout, so one stuck customer can't abort the rest of a batch update - repair_container_naming() self-heals orphaned hash-renamed containers by renaming them back before every status check and before recreate - update-all loop now catches per-customer exceptions instead of aborting - status responses expose "unknown" separately from "needs_update" so the UI shows a distinct grey badge instead of a false-positive green one - new settings: automatic daily update check (on/off + time), with an independent toggle for whether it also auto-recreates customer containers
2026-08-19 09:10:29 +02:00
parent 0e38b8083c
commit e53539231e
11 changed files with 359 additions and 18 deletions
@@ -0,0 +1,119 @@
+"""Background scheduler for automatic NetBird image update checks.
+
+No external scheduler dependency (APScheduler etc.) — a single asyncio task
+started at app startup wakes up once a minute, and only actually does
+anything once per day at the configured HH:MM, controlled entirely by
+SystemConfig.auto_update_check_enabled / auto_update_check_time.
+"""
+
+import asyncio
+import logging
+from datetime import datetime
+
+from app.database import SessionLocal
+from app.models import Deployment, SystemConfig
+from app.services import image_service
+
+logger = logging.getLogger(__name__)
+
+_POLL_INTERVAL_SECONDS = 60
+_task: asyncio.Task | None = None
+
+
+def start() -> None:
+    """Start the background polling task. Safe to call once at app startup."""
+    global _task
+    if _task is None or _task.done():
+        _task = asyncio.create_task(_poll_loop())
+        logger.info("Automatic update scheduler started.")
+
+
+def stop() -> None:
+    """Cancel the background polling task."""
+    global _task
+    if _task is not None:
+        _task.cancel()
+        _task = None
+
+
+async def _poll_loop() -> None:
+    while True:
+        try:
+            await _tick()
+        except Exception:
+            logger.exception("Scheduler tick failed")
+        await asyncio.sleep(_POLL_INTERVAL_SECONDS)
+
+
+async def _tick() -> None:
+    db = SessionLocal()
+    try:
+        config = db.query(SystemConfig).filter(SystemConfig.id == 1).first()
+        if not config or not config.auto_update_check_enabled:
+            return
+
+        now = datetime.now()
+        target_time = config.auto_update_check_time or "03:00"
+        current_hhmm = now.strftime("%H:%M")
+        if current_hhmm != target_time:
+            return
+
+        last_run = config.auto_update_last_run_at
+        if last_run and last_run.date() == now.date():
+            return  # already ran today
+
+        # Claim this run immediately so a slow run can't overlap the next tick.
+        config.auto_update_last_run_at = now
+        db.commit()
+
+        apply_enabled = bool(config.auto_update_apply_enabled)
+        logger.info(
+            "Running scheduled NetBird image update check (auto-apply=%s)...", apply_enabled
+        )
+        await _run_check_and_optionally_apply(config, apply_enabled)
+    finally:
+        db.close()
+
+
+async def _run_check_and_optionally_apply(config: SystemConfig, apply_enabled: bool) -> None:
+    hub_status = await image_service.check_all_images(config)
+    if not hub_status["any_update_available"]:
+        logger.info("Scheduled check: all NetBird images already up to date.")
+        return
+
+    logger.info("Scheduled check: new NetBird image(s) available — pulling.")
+    pull_result = await image_service.pull_all_images(config)
+    if not pull_result["all_success"]:
+        logger.error("Scheduled image pull had failures: %s", pull_result["results"])
+
+    if not apply_enabled:
+        logger.info("Auto-apply disabled — images pulled, customer containers left untouched.")
+        return
+
+    db = SessionLocal()
+    try:
+        deployments = db.query(Deployment).all()
+        to_update = []
+        for dep in deployments:
+            cs = image_service.get_customer_container_image_status(dep.container_prefix, config)
+            if cs["needs_update"]:
+                customer = dep.customer
+                to_update.append({
+                    "instance_dir": f"{config.data_dir}/{customer.subdomain}",
+                    "project_name": dep.container_prefix,
+                    "customer_name": customer.name,
+                })
+        logger.info("Scheduled auto-apply: updating %d customer(s)...", len(to_update))
+        for entry in to_update:
+            try:
+                res = await image_service.update_customer_containers(
+                    entry["instance_dir"], entry["project_name"]
+                )
+                logger.info(
+                    "Scheduled update for %s: %s",
+                    entry["customer_name"], "OK" if res["success"] else res.get("error"),
+                )
+            except Exception:
+                logger.exception("Scheduled update failed for %s", entry["customer_name"])
+    finally:
+        db.close()