feat(monitoring): add automatic restart of unreachable customer containers

Some customers only regain access to their NetBird web UI after a manual
container restart, and until now that required someone to notice and do it
by hand. The scheduler now optionally probes each customer's actual
dashboard URL over HTTP(S) every poll interval (not just Docker container
state, since a container can be "running" while the web UI behind it still
doesn't respond) and restarts that customer's containers if it doesn't
answer, gated by a per-deployment cooldown to avoid restart loops on a
persistently broken customer.

New system setting "Automatic Restart of Unreachable Containers" (default
off) with a configurable cooldown. Every automatic restart is logged in the
affected customer's own log, same as a manual restart.

Co-Authored-By: Claude Sonnet 5 <[email protected]>

app/database.py

@@ -133,6 +133,10 @@ def _run_migrations() -> None:
         ("deployments", "netbird_api_token_renewed_at", "TEXT"),
         ("system_config", "netbird_client_auto_update_version", "TEXT DEFAULT 'disabled'"),
         ("system_config", "netbird_client_auto_update_always", "BOOLEAN DEFAULT 0"),
+        # Automatic restart of unreachable customer containers
+        ("system_config", "auto_restart_unreachable_enabled", "BOOLEAN DEFAULT 0"),
+        ("system_config", "auto_restart_cooldown_minutes", "INTEGER DEFAULT 30"),
+        ("deployments", "last_auto_restart_at", "TEXT"),
     ]
     for table, column, col_type in migrations:
         if not _has_column(table, column):

app/models.py

@@ -96,6 +96,7 @@ class Deployment(Base):
     )
     deployed_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
     last_health_check: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
+    last_auto_restart_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
 
     __table_args__ = (
         CheckConstraint(
@@ -128,6 +129,9 @@ class Deployment(Base):
             "last_health_check": (
                 self.last_health_check.isoformat() if self.last_health_check else None
             ),
+            "last_auto_restart_at": (
+                self.last_auto_restart_at.isoformat() if self.last_auto_restart_at else None
+            ),
         }
 
 
@@ -212,6 +216,10 @@ class SystemConfig(Base):
     auto_update_apply_enabled: Mapped[bool] = mapped_column(Boolean, default=False)
     auto_update_last_run_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
 
+    # Automatic restart of customer containers when found unreachable/not running
+    auto_restart_unreachable_enabled: Mapped[bool] = mapped_column(Boolean, default=False)
+    auto_restart_cooldown_minutes: Mapped[int] = mapped_column(Integer, default=30)
+
     # Master default for the NetBird *client* (peer) automatic-updates feature
     # (Settings > Clients > Automatic Updates inside each customer's own
     # NetBird dashboard) — pushed to customers via the NetBird Management API.
@@ -278,6 +286,8 @@ class SystemConfig(Base):
             "auto_update_last_run_at": (
                 self.auto_update_last_run_at.isoformat() if self.auto_update_last_run_at else None
             ),
+            "auto_restart_unreachable_enabled": bool(self.auto_restart_unreachable_enabled),
+            "auto_restart_cooldown_minutes": self.auto_restart_cooldown_minutes or 30,
             "netbird_client_auto_update_version": self.netbird_client_auto_update_version or "disabled",
             "netbird_client_auto_update_always": bool(self.netbird_client_auto_update_always),
             "created_at": self.created_at.isoformat() if self.created_at else None,

app/services/netbird_service.py

@@ -26,6 +26,7 @@ import urllib.error
 from datetime import datetime
 from typing import Any
 
+import httpx
 from jinja2 import Environment, FileSystemLoader
 from sqlalchemy.orm import Session
 
@@ -529,6 +530,25 @@ async def restart_customer(db: Session, customer_id: int) -> dict[str, Any]:
     return {"success": ok}
 
 
+async def check_dashboard_reachable(setup_url: str, timeout: float = 10) -> bool:
+    """Check whether a customer's NetBird dashboard actually responds over HTTP(S).
+
+    A container can be "running" in Docker while the web UI behind it is
+    still unreachable (e.g. the management service is up but not accepting
+    connections yet), so this checks the real endpoint rather than the
+    container state. Any HTTP response (including 4xx/5xx from the app
+    itself) counts as reachable — only connection failures/timeouts don't.
+    """
+    if not setup_url:
+        return True  # nothing to check (e.g. deployment not fully set up yet)
+    try:
+        async with httpx.AsyncClient(timeout=timeout) as client:
+            await client.get(setup_url)
+        return True
+    except httpx.HTTPError:
+        return False
+
+
 def get_customer_health(db: Session, customer_id: int) -> dict[str, Any]:
     """Check health of a customer's deployment."""
     deployment = db.query(Deployment).filter(Deployment.customer_id == customer_id).first()

app/services/scheduler_service.py

@@ -12,7 +12,14 @@ from datetime import datetime, timedelta
 
 from app.database import SessionLocal
 from app.models import Deployment, SystemConfig
-from app.services import image_service, netbird_client_update_service, update_log_service
+from app.services import (
+    docker_service,
+    image_service,
+    netbird_client_update_service,
+    netbird_service,
+    update_log_service,
+)
+from app.services.netbird_service import _log_action
 from app.utils.security import decrypt_value, encrypt_value
 
 logger = logging.getLogger(__name__)
@@ -55,6 +62,10 @@ async def _poll_loop() -> None:
             await _token_renewal_tick()
         except Exception:
             logger.exception("Token renewal tick failed")
+        try:
+            await _auto_restart_tick()
+        except Exception:
+            logger.exception("Auto-restart tick failed")
         await asyncio.sleep(_POLL_INTERVAL_SECONDS)
 
 
@@ -208,3 +219,65 @@ async def _run_check_and_optionally_apply(config: SystemConfig, apply_enabled: b
         )
     finally:
         log_db.close()
+
+
+async def _auto_restart_tick() -> None:
+    """Every poll interval, restart any customer whose NetBird dashboard
+    doesn't actually respond over HTTP(S) — a container can be "running" in
+    Docker while the web UI behind it is still unreachable, which is the
+    actual symptom customers hit, so this probes the real endpoint rather
+    than trusting container state alone. Falls back to container status
+    only if a customer has no setup_url yet. Guarded by
+    SystemConfig.auto_restart_unreachable_enabled and a per-deployment
+    cooldown so a persistently broken customer doesn't get restarted in a
+    tight loop, and skipped for deployments that aren't running/failed
+    (e.g. intentionally stopped or still pending).
+    """
+    db = SessionLocal()
+    try:
+        config = db.query(SystemConfig).filter(SystemConfig.id == 1).first()
+        if not config or not config.auto_restart_unreachable_enabled:
+            return
+
+        cooldown = timedelta(minutes=config.auto_restart_cooldown_minutes or 30)
+        now = datetime.now()
+
+        deployments = db.query(Deployment).all()
+        for dep in deployments:
+            if dep.deployment_status not in ("running", "failed"):
+                continue
+            if dep.last_auto_restart_at and now - dep.last_auto_restart_at < cooldown:
+                continue
+
+            reason = None
+            if dep.setup_url:
+                if not await netbird_service.check_dashboard_reachable(dep.setup_url):
+                    reason = f"dashboard ({dep.setup_url}) did not respond"
+            else:
+                containers = await docker_service.get_container_status_async(dep.container_prefix)
+                if not containers or not all(c["status"] == "running" for c in containers):
+                    not_running = [c["name"] for c in containers if c["status"] != "running"] or ["no containers found"]
+                    reason = f"container(s) not running: {', '.join(not_running)}"
+
+            if not reason:
+                continue
+
+            logger.warning(
+                "Auto-restart: customer %s unreachable (%s) — restarting.",
+                dep.container_prefix, reason,
+            )
+            _log_action(
+                db, dep.customer_id, "auto_restart", "info",
+                f"Automatic restart triggered: {reason}.",
+            )
+            try:
+                result = await netbird_service.restart_customer(db, dep.customer_id)
+                if not result["success"]:
+                    logger.error("Auto-restart failed for customer %s.", dep.container_prefix)
+            except Exception:
+                logger.exception("Auto-restart failed for customer %s.", dep.container_prefix)
+
+            dep.last_auto_restart_at = now
+            db.commit()
+    finally:
+        db.close()

app/utils/validators.py

@@ -165,6 +165,9 @@ class SystemConfigUpdate(BaseModel):
     # Master default for the NetBird client (peer) automatic-updates feature
     netbird_client_auto_update_version: Optional[str] = Field(None, max_length=50)
     netbird_client_auto_update_always: Optional[bool] = None
+    # Automatic restart of unreachable customer containers
+    auto_restart_unreachable_enabled: Optional[bool] = None
+    auto_restart_cooldown_minutes: Optional[int] = Field(None, ge=1, le=1440)
 
     @field_validator("auto_update_check_time")
     @classmethod