diff --git a/app/database.py b/app/database.py
index 45ad713..46f4b06 100644
--- a/app/database.py
+++ b/app/database.py
@@ -133,6 +133,10 @@ def _run_migrations() -> None:
("deployments", "netbird_api_token_renewed_at", "TEXT"),
("system_config", "netbird_client_auto_update_version", "TEXT DEFAULT 'disabled'"),
("system_config", "netbird_client_auto_update_always", "BOOLEAN DEFAULT 0"),
+ # Automatic restart of unreachable customer containers
+ ("system_config", "auto_restart_unreachable_enabled", "BOOLEAN DEFAULT 0"),
+ ("system_config", "auto_restart_cooldown_minutes", "INTEGER DEFAULT 30"),
+ ("deployments", "last_auto_restart_at", "TEXT"),
]
for table, column, col_type in migrations:
if not _has_column(table, column):
diff --git a/app/models.py b/app/models.py
index 97f689f..a49a9bf 100644
--- a/app/models.py
+++ b/app/models.py
@@ -96,6 +96,7 @@ class Deployment(Base):
)
deployed_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
last_health_check: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
+ last_auto_restart_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
__table_args__ = (
CheckConstraint(
@@ -128,6 +129,9 @@ class Deployment(Base):
"last_health_check": (
self.last_health_check.isoformat() if self.last_health_check else None
),
+ "last_auto_restart_at": (
+ self.last_auto_restart_at.isoformat() if self.last_auto_restart_at else None
+ ),
}
@@ -212,6 +216,10 @@ class SystemConfig(Base):
auto_update_apply_enabled: Mapped[bool] = mapped_column(Boolean, default=False)
auto_update_last_run_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
+ # Automatic restart of customer containers when found unreachable/not running
+ auto_restart_unreachable_enabled: Mapped[bool] = mapped_column(Boolean, default=False)
+ auto_restart_cooldown_minutes: Mapped[int] = mapped_column(Integer, default=30)
+
# Master default for the NetBird *client* (peer) automatic-updates feature
# (Settings > Clients > Automatic Updates inside each customer's own
# NetBird dashboard) — pushed to customers via the NetBird Management API.
@@ -278,6 +286,8 @@ class SystemConfig(Base):
"auto_update_last_run_at": (
self.auto_update_last_run_at.isoformat() if self.auto_update_last_run_at else None
),
+ "auto_restart_unreachable_enabled": bool(self.auto_restart_unreachable_enabled),
+ "auto_restart_cooldown_minutes": self.auto_restart_cooldown_minutes or 30,
"netbird_client_auto_update_version": self.netbird_client_auto_update_version or "disabled",
"netbird_client_auto_update_always": bool(self.netbird_client_auto_update_always),
"created_at": self.created_at.isoformat() if self.created_at else None,
diff --git a/app/services/netbird_service.py b/app/services/netbird_service.py
index 57c9b33..7e34d88 100644
--- a/app/services/netbird_service.py
+++ b/app/services/netbird_service.py
@@ -26,6 +26,7 @@ import urllib.error
from datetime import datetime
from typing import Any
+import httpx
from jinja2 import Environment, FileSystemLoader
from sqlalchemy.orm import Session
@@ -529,6 +530,25 @@ async def restart_customer(db: Session, customer_id: int) -> dict[str, Any]:
return {"success": ok}
+async def check_dashboard_reachable(setup_url: str, timeout: float = 10) -> bool:
+ """Check whether a customer's NetBird dashboard actually responds over HTTP(S).
+
+ A container can be "running" in Docker while the web UI behind it is
+ still unreachable (e.g. the management service is up but not accepting
+ connections yet), so this checks the real endpoint rather than the
+ container state. Any HTTP response (including 4xx/5xx from the app
+ itself) counts as reachable — only connection failures/timeouts don't.
+ """
+ if not setup_url:
+ return True # nothing to check (e.g. deployment not fully set up yet)
+ try:
+ async with httpx.AsyncClient(timeout=timeout) as client:
+ await client.get(setup_url)
+ return True
+ except httpx.HTTPError:
+ return False
+
+
def get_customer_health(db: Session, customer_id: int) -> dict[str, Any]:
"""Check health of a customer's deployment."""
deployment = db.query(Deployment).filter(Deployment.customer_id == customer_id).first()
diff --git a/app/services/scheduler_service.py b/app/services/scheduler_service.py
index 2d83927..f6fd896 100644
--- a/app/services/scheduler_service.py
+++ b/app/services/scheduler_service.py
@@ -12,7 +12,14 @@ from datetime import datetime, timedelta
from app.database import SessionLocal
from app.models import Deployment, SystemConfig
-from app.services import image_service, netbird_client_update_service, update_log_service
+from app.services import (
+ docker_service,
+ image_service,
+ netbird_client_update_service,
+ netbird_service,
+ update_log_service,
+)
+from app.services.netbird_service import _log_action
from app.utils.security import decrypt_value, encrypt_value
logger = logging.getLogger(__name__)
@@ -55,6 +62,10 @@ async def _poll_loop() -> None:
await _token_renewal_tick()
except Exception:
logger.exception("Token renewal tick failed")
+ try:
+ await _auto_restart_tick()
+ except Exception:
+ logger.exception("Auto-restart tick failed")
await asyncio.sleep(_POLL_INTERVAL_SECONDS)
@@ -208,3 +219,65 @@ async def _run_check_and_optionally_apply(config: SystemConfig, apply_enabled: b
)
finally:
log_db.close()
+
+
+async def _auto_restart_tick() -> None:
+ """Every poll interval, restart any customer whose NetBird dashboard
+ doesn't actually respond over HTTP(S) — a container can be "running" in
+ Docker while the web UI behind it is still unreachable, which is the
+ actual symptom customers hit, so this probes the real endpoint rather
+ than trusting container state alone. Falls back to container status
+ only if a customer has no setup_url yet. Guarded by
+ SystemConfig.auto_restart_unreachable_enabled and a per-deployment
+ cooldown so a persistently broken customer doesn't get restarted in a
+ tight loop, and skipped for deployments that aren't running/failed
+ (e.g. intentionally stopped or still pending).
+ """
+ db = SessionLocal()
+ try:
+ config = db.query(SystemConfig).filter(SystemConfig.id == 1).first()
+ if not config or not config.auto_restart_unreachable_enabled:
+ return
+
+ cooldown = timedelta(minutes=config.auto_restart_cooldown_minutes or 30)
+ now = datetime.now()
+
+ deployments = db.query(Deployment).all()
+ for dep in deployments:
+ if dep.deployment_status not in ("running", "failed"):
+ continue
+ if dep.last_auto_restart_at and now - dep.last_auto_restart_at < cooldown:
+ continue
+
+ reason = None
+ if dep.setup_url:
+ if not await netbird_service.check_dashboard_reachable(dep.setup_url):
+ reason = f"dashboard ({dep.setup_url}) did not respond"
+ else:
+ containers = await docker_service.get_container_status_async(dep.container_prefix)
+ if not containers or not all(c["status"] == "running" for c in containers):
+ not_running = [c["name"] for c in containers if c["status"] != "running"] or ["no containers found"]
+ reason = f"container(s) not running: {', '.join(not_running)}"
+
+ if not reason:
+ continue
+
+ logger.warning(
+ "Auto-restart: customer %s unreachable (%s) — restarting.",
+ dep.container_prefix, reason,
+ )
+ _log_action(
+ db, dep.customer_id, "auto_restart", "info",
+ f"Automatic restart triggered: {reason}.",
+ )
+ try:
+ result = await netbird_service.restart_customer(db, dep.customer_id)
+ if not result["success"]:
+ logger.error("Auto-restart failed for customer %s.", dep.container_prefix)
+ except Exception:
+ logger.exception("Auto-restart failed for customer %s.", dep.container_prefix)
+
+ dep.last_auto_restart_at = now
+ db.commit()
+ finally:
+ db.close()
diff --git a/app/utils/validators.py b/app/utils/validators.py
index 09bec91..7e8c116 100644
--- a/app/utils/validators.py
+++ b/app/utils/validators.py
@@ -165,6 +165,9 @@ class SystemConfigUpdate(BaseModel):
# Master default for the NetBird client (peer) automatic-updates feature
netbird_client_auto_update_version: Optional[str] = Field(None, max_length=50)
netbird_client_auto_update_always: Optional[bool] = None
+ # Automatic restart of unreachable customer containers
+ auto_restart_unreachable_enabled: Optional[bool] = None
+ auto_restart_cooldown_minutes: Optional[int] = Field(None, ge=1, le=1440)
@field_validator("auto_update_check_time")
@classmethod
diff --git a/static/index.html b/static/index.html
index 3d29b51..fc92817 100644
--- a/static/index.html
+++ b/static/index.html
@@ -669,6 +669,24 @@
+ Automatic Restart of Unreachable Containers
+ Some customers only regain access to their NetBird web UI after a container restart. When enabled, the scheduler periodically checks whether each customer's dashboard actually responds and restarts their containers if it doesn't, logging the action in the customer's log.
+
+
NetBird Client Auto-Updates (all customers)
Controls the "Automatic Updates" setting inside every customer's own NetBird dashboard (Settings > Clients). Set the default here, then push it to all customers at once. Individual customers can still be overridden from their detail page.