From 189c49d505c460ba984739f9b300569456b341cd Mon Sep 17 00:00:00 2001 From: twothatIT Date: Wed, 23 Sep 2026 13:20:40 +0200 Subject: [PATCH] feat(monitoring): add automatic restart of unreachable customer containers Some customers only regain access to their NetBird web UI after a manual container restart, and until now that required someone to notice and do it by hand. The scheduler now optionally probes each customer's actual dashboard URL over HTTP(S) every poll interval (not just Docker container state, since a container can be "running" while the web UI behind it still doesn't respond) and restarts that customer's containers if it doesn't answer, gated by a per-deployment cooldown to avoid restart loops on a persistently broken customer. New system setting "Automatic Restart of Unreachable Containers" (default off) with a configurable cooldown. Every automatic restart is logged in the affected customer's own log, same as a manual restart. Co-Authored-By: Claude Sonnet 5 --- app/database.py | 4 ++ app/models.py | 10 +++++ app/services/netbird_service.py | 20 +++++++++ app/services/scheduler_service.py | 75 ++++++++++++++++++++++++++++++- app/utils/validators.py | 3 ++ static/index.html | 18 ++++++++ static/js/app.js | 17 +++++++ static/lang/de.json | 5 +++ static/lang/en.json | 5 +++ 9 files changed, 156 insertions(+), 1 deletion(-) diff --git a/app/database.py b/app/database.py index 45ad713..46f4b06 100644 --- a/app/database.py +++ b/app/database.py @@ -133,6 +133,10 @@ def _run_migrations() -> None: ("deployments", "netbird_api_token_renewed_at", "TEXT"), ("system_config", "netbird_client_auto_update_version", "TEXT DEFAULT 'disabled'"), ("system_config", "netbird_client_auto_update_always", "BOOLEAN DEFAULT 0"), + # Automatic restart of unreachable customer containers + ("system_config", "auto_restart_unreachable_enabled", "BOOLEAN DEFAULT 0"), + ("system_config", "auto_restart_cooldown_minutes", "INTEGER DEFAULT 30"), + ("deployments", "last_auto_restart_at", "TEXT"), ] for table, column, col_type in migrations: if not _has_column(table, column): diff --git a/app/models.py b/app/models.py index 97f689f..a49a9bf 100644 --- a/app/models.py +++ b/app/models.py @@ -96,6 +96,7 @@ class Deployment(Base): ) deployed_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) last_health_check: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True) + last_auto_restart_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True) __table_args__ = ( CheckConstraint( @@ -128,6 +129,9 @@ class Deployment(Base): "last_health_check": ( self.last_health_check.isoformat() if self.last_health_check else None ), + "last_auto_restart_at": ( + self.last_auto_restart_at.isoformat() if self.last_auto_restart_at else None + ), } @@ -212,6 +216,10 @@ class SystemConfig(Base): auto_update_apply_enabled: Mapped[bool] = mapped_column(Boolean, default=False) auto_update_last_run_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True) + # Automatic restart of customer containers when found unreachable/not running + auto_restart_unreachable_enabled: Mapped[bool] = mapped_column(Boolean, default=False) + auto_restart_cooldown_minutes: Mapped[int] = mapped_column(Integer, default=30) + # Master default for the NetBird *client* (peer) automatic-updates feature # (Settings > Clients > Automatic Updates inside each customer's own # NetBird dashboard) — pushed to customers via the NetBird Management API. @@ -278,6 +286,8 @@ class SystemConfig(Base): "auto_update_last_run_at": ( self.auto_update_last_run_at.isoformat() if self.auto_update_last_run_at else None ), + "auto_restart_unreachable_enabled": bool(self.auto_restart_unreachable_enabled), + "auto_restart_cooldown_minutes": self.auto_restart_cooldown_minutes or 30, "netbird_client_auto_update_version": self.netbird_client_auto_update_version or "disabled", "netbird_client_auto_update_always": bool(self.netbird_client_auto_update_always), "created_at": self.created_at.isoformat() if self.created_at else None, diff --git a/app/services/netbird_service.py b/app/services/netbird_service.py index 57c9b33..7e34d88 100644 --- a/app/services/netbird_service.py +++ b/app/services/netbird_service.py @@ -26,6 +26,7 @@ import urllib.error from datetime import datetime from typing import Any +import httpx from jinja2 import Environment, FileSystemLoader from sqlalchemy.orm import Session @@ -529,6 +530,25 @@ async def restart_customer(db: Session, customer_id: int) -> dict[str, Any]: return {"success": ok} +async def check_dashboard_reachable(setup_url: str, timeout: float = 10) -> bool: + """Check whether a customer's NetBird dashboard actually responds over HTTP(S). + + A container can be "running" in Docker while the web UI behind it is + still unreachable (e.g. the management service is up but not accepting + connections yet), so this checks the real endpoint rather than the + container state. Any HTTP response (including 4xx/5xx from the app + itself) counts as reachable — only connection failures/timeouts don't. + """ + if not setup_url: + return True # nothing to check (e.g. deployment not fully set up yet) + try: + async with httpx.AsyncClient(timeout=timeout) as client: + await client.get(setup_url) + return True + except httpx.HTTPError: + return False + + def get_customer_health(db: Session, customer_id: int) -> dict[str, Any]: """Check health of a customer's deployment.""" deployment = db.query(Deployment).filter(Deployment.customer_id == customer_id).first() diff --git a/app/services/scheduler_service.py b/app/services/scheduler_service.py index 2d83927..f6fd896 100644 --- a/app/services/scheduler_service.py +++ b/app/services/scheduler_service.py @@ -12,7 +12,14 @@ from datetime import datetime, timedelta from app.database import SessionLocal from app.models import Deployment, SystemConfig -from app.services import image_service, netbird_client_update_service, update_log_service +from app.services import ( + docker_service, + image_service, + netbird_client_update_service, + netbird_service, + update_log_service, +) +from app.services.netbird_service import _log_action from app.utils.security import decrypt_value, encrypt_value logger = logging.getLogger(__name__) @@ -55,6 +62,10 @@ async def _poll_loop() -> None: await _token_renewal_tick() except Exception: logger.exception("Token renewal tick failed") + try: + await _auto_restart_tick() + except Exception: + logger.exception("Auto-restart tick failed") await asyncio.sleep(_POLL_INTERVAL_SECONDS) @@ -208,3 +219,65 @@ async def _run_check_and_optionally_apply(config: SystemConfig, apply_enabled: b ) finally: log_db.close() + + +async def _auto_restart_tick() -> None: + """Every poll interval, restart any customer whose NetBird dashboard + doesn't actually respond over HTTP(S) — a container can be "running" in + Docker while the web UI behind it is still unreachable, which is the + actual symptom customers hit, so this probes the real endpoint rather + than trusting container state alone. Falls back to container status + only if a customer has no setup_url yet. Guarded by + SystemConfig.auto_restart_unreachable_enabled and a per-deployment + cooldown so a persistently broken customer doesn't get restarted in a + tight loop, and skipped for deployments that aren't running/failed + (e.g. intentionally stopped or still pending). + """ + db = SessionLocal() + try: + config = db.query(SystemConfig).filter(SystemConfig.id == 1).first() + if not config or not config.auto_restart_unreachable_enabled: + return + + cooldown = timedelta(minutes=config.auto_restart_cooldown_minutes or 30) + now = datetime.now() + + deployments = db.query(Deployment).all() + for dep in deployments: + if dep.deployment_status not in ("running", "failed"): + continue + if dep.last_auto_restart_at and now - dep.last_auto_restart_at < cooldown: + continue + + reason = None + if dep.setup_url: + if not await netbird_service.check_dashboard_reachable(dep.setup_url): + reason = f"dashboard ({dep.setup_url}) did not respond" + else: + containers = await docker_service.get_container_status_async(dep.container_prefix) + if not containers or not all(c["status"] == "running" for c in containers): + not_running = [c["name"] for c in containers if c["status"] != "running"] or ["no containers found"] + reason = f"container(s) not running: {', '.join(not_running)}" + + if not reason: + continue + + logger.warning( + "Auto-restart: customer %s unreachable (%s) — restarting.", + dep.container_prefix, reason, + ) + _log_action( + db, dep.customer_id, "auto_restart", "info", + f"Automatic restart triggered: {reason}.", + ) + try: + result = await netbird_service.restart_customer(db, dep.customer_id) + if not result["success"]: + logger.error("Auto-restart failed for customer %s.", dep.container_prefix) + except Exception: + logger.exception("Auto-restart failed for customer %s.", dep.container_prefix) + + dep.last_auto_restart_at = now + db.commit() + finally: + db.close() diff --git a/app/utils/validators.py b/app/utils/validators.py index 09bec91..7e8c116 100644 --- a/app/utils/validators.py +++ b/app/utils/validators.py @@ -165,6 +165,9 @@ class SystemConfigUpdate(BaseModel): # Master default for the NetBird client (peer) automatic-updates feature netbird_client_auto_update_version: Optional[str] = Field(None, max_length=50) netbird_client_auto_update_always: Optional[bool] = None + # Automatic restart of unreachable customer containers + auto_restart_unreachable_enabled: Optional[bool] = None + auto_restart_cooldown_minutes: Optional[int] = Field(None, ge=1, le=1440) @field_validator("auto_update_check_time") @classmethod diff --git a/static/index.html b/static/index.html index 3d29b51..fc92817 100644 --- a/static/index.html +++ b/static/index.html @@ -669,6 +669,24 @@
+
Automatic Restart of Unreachable Containers
+

Some customers only regain access to their NetBird web UI after a container restart. When enabled, the scheduler periodically checks whether each customer's dashboard actually responds and restarts their containers if it doesn't, logging the action in the customer's log.

+
+
+ + +
+
+
+ + +
+
+
+ +
+
+
NetBird Client Auto-Updates (all customers)

Controls the "Automatic Updates" setting inside every customer's own NetBird dashboard (Settings > Clients). Set the default here, then push it to all customers at once. Individual customers can still be overridden from their detail page.

diff --git a/static/js/app.js b/static/js/app.js index e9f2938..2112f8f 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -1096,6 +1096,9 @@ async function loadSettings() { ? new Date(cfg.auto_update_last_run_at).toLocaleString() : t('monitoring.autoUpdateNever'); + document.getElementById('cfg-auto-restart-enabled').checked = cfg.auto_restart_unreachable_enabled || false; + document.getElementById('cfg-auto-restart-cooldown').value = cfg.auto_restart_cooldown_minutes || 30; + const nbuVersion = cfg.netbird_client_auto_update_version || 'disabled'; const nbuIsCustom = nbuVersion !== 'disabled' && nbuVersion !== 'latest'; document.getElementById('cfg-nbu-version-select').value = nbuIsCustom ? 'custom' : nbuVersion; @@ -1236,6 +1239,20 @@ document.getElementById('settings-auto-update-form').addEventListener('submit', } }); +// Automatic restart of unreachable customer containers +document.getElementById('settings-auto-restart-form').addEventListener('submit', async (e) => { + e.preventDefault(); + try { + await api('PUT', '/settings/system', { + auto_restart_unreachable_enabled: document.getElementById('cfg-auto-restart-enabled').checked, + auto_restart_cooldown_minutes: parseInt(document.getElementById('cfg-auto-restart-cooldown').value, 10) || 30, + }); + showSettingsAlert('success', t('messages.imageSettingsSaved')); + } catch (err) { + showSettingsAlert('danger', t('errors.failed', { error: err.message })); + } +}); + function _readNbuMasterForm() { const select = document.getElementById('cfg-nbu-version-select').value; const version = select === 'custom' ? document.getElementById('cfg-nbu-custom-version').value.trim() : select; diff --git a/static/lang/de.json b/static/lang/de.json index 2881c80..0c9c8f4 100644 --- a/static/lang/de.json +++ b/static/lang/de.json @@ -455,6 +455,11 @@ "autoUpdateLastRun": "Letzte automatische Prüfung", "autoUpdateNever": "Noch nie ausgeführt", "saveAutoUpdateSettings": "Automatisierung speichern", + "autoRestartTitle": "Automatischer Neustart nicht erreichbarer Container", + "autoRestartHint": "Manche Kunden erhalten den Zugriff auf ihre NetBird-Weboberfläche erst nach einem Container-Neustart zurück. Wenn aktiviert, prüft der Scheduler regelmäßig, ob die Weboberfläche jedes Kunden tatsächlich antwortet, und startet die Container bei Bedarf neu — mit Protokollierung im Kunden-Log.", + "autoRestartEnabled": "Nicht erreichbare Kunden-Container automatisch neu starten", + "autoRestartCooldown": "Wartezeit zwischen Neustarts (Minuten)", + "saveAutoRestartSettings": "Automatisierung speichern", "updateLogTitle": "Update-Verlauf", "noUpdateLogs": "Noch keine Update-Läufe protokolliert.", "logTypeScheduled": "Automatisch", diff --git a/static/lang/en.json b/static/lang/en.json index 72eab7b..e7e9a07 100644 --- a/static/lang/en.json +++ b/static/lang/en.json @@ -362,6 +362,11 @@ "autoUpdateLastRun": "Last automatic check", "autoUpdateNever": "Never run", "saveAutoUpdateSettings": "Save Automation", + "autoRestartTitle": "Automatic Restart of Unreachable Containers", + "autoRestartHint": "Some customers only regain access to their NetBird web UI after a container restart. When enabled, the scheduler periodically checks whether each customer's dashboard actually responds and restarts their containers if it doesn't, logging the action in the customer's log.", + "autoRestartEnabled": "Automatically restart unreachable customer containers", + "autoRestartCooldown": "Cooldown between restarts (minutes)", + "saveAutoRestartSettings": "Save Automation", "updateLogTitle": "Update Log", "noUpdateLogs": "No update runs logged yet.", "logTypeScheduled": "Automatic",