chore(release): bump version to 1.6.1

Co-Authored-By: Claude Sonnet 5 <[email protected]>

app/database.py

@@ -133,6 +133,10 @@ def _run_migrations() -> None:
         ("deployments", "netbird_api_token_renewed_at", "TEXT"),
         ("system_config", "netbird_client_auto_update_version", "TEXT DEFAULT 'disabled'"),
         ("system_config", "netbird_client_auto_update_always", "BOOLEAN DEFAULT 0"),
+        # Automatic restart of unreachable customer containers
+        ("system_config", "auto_restart_unreachable_enabled", "BOOLEAN DEFAULT 0"),
+        ("system_config", "auto_restart_cooldown_minutes", "INTEGER DEFAULT 30"),
+        ("deployments", "last_auto_restart_at", "TEXT"),
     ]
     for table, column, col_type in migrations:
         if not _has_column(table, column):

app/main.py

@@ -34,7 +34,7 @@ logger = logging.getLogger(__name__)
 app = FastAPI(
     title="NetBird MSP Appliance",
     description="Multi-tenant NetBird management platform for MSPs",
-    version="1.5.1",
+    version="1.6.1",
     docs_url="/api/docs",
     redoc_url="/api/redoc",
     openapi_url="/api/openapi.json",

app/models.py

@@ -96,6 +96,7 @@ class Deployment(Base):
     )
     deployed_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
     last_health_check: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
+    last_auto_restart_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
 
     __table_args__ = (
         CheckConstraint(
@@ -128,6 +129,9 @@ class Deployment(Base):
             "last_health_check": (
                 self.last_health_check.isoformat() if self.last_health_check else None
             ),
+            "last_auto_restart_at": (
+                self.last_auto_restart_at.isoformat() if self.last_auto_restart_at else None
+            ),
         }
 
 
@@ -212,6 +216,10 @@ class SystemConfig(Base):
     auto_update_apply_enabled: Mapped[bool] = mapped_column(Boolean, default=False)
     auto_update_last_run_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
 
+    # Automatic restart of customer containers when found unreachable/not running
+    auto_restart_unreachable_enabled: Mapped[bool] = mapped_column(Boolean, default=False)
+    auto_restart_cooldown_minutes: Mapped[int] = mapped_column(Integer, default=30)
+
     # Master default for the NetBird *client* (peer) automatic-updates feature
     # (Settings > Clients > Automatic Updates inside each customer's own
     # NetBird dashboard) — pushed to customers via the NetBird Management API.
@@ -278,6 +286,8 @@ class SystemConfig(Base):
             "auto_update_last_run_at": (
                 self.auto_update_last_run_at.isoformat() if self.auto_update_last_run_at else None
             ),
+            "auto_restart_unreachable_enabled": bool(self.auto_restart_unreachable_enabled),
+            "auto_restart_cooldown_minutes": self.auto_restart_cooldown_minutes or 30,
             "netbird_client_auto_update_version": self.netbird_client_auto_update_version or "disabled",
             "netbird_client_auto_update_always": bool(self.netbird_client_auto_update_always),
             "created_at": self.created_at.isoformat() if self.created_at else None,

app/services/netbird_service.py

@@ -26,6 +26,7 @@ import urllib.error
 from datetime import datetime
 from typing import Any
 
+import httpx
 from jinja2 import Environment, FileSystemLoader
 from sqlalchemy.orm import Session
 
@@ -529,6 +530,25 @@ async def restart_customer(db: Session, customer_id: int) -> dict[str, Any]:
     return {"success": ok}
 
 
+async def check_dashboard_reachable(setup_url: str, timeout: float = 10) -> bool:
+    """Check whether a customer's NetBird dashboard actually responds over HTTP(S).
+
+    A container can be "running" in Docker while the web UI behind it is
+    still unreachable (e.g. the management service is up but not accepting
+    connections yet), so this checks the real endpoint rather than the
+    container state. Any HTTP response (including 4xx/5xx from the app
+    itself) counts as reachable — only connection failures/timeouts don't.
+    """
+    if not setup_url:
+        return True  # nothing to check (e.g. deployment not fully set up yet)
+    try:
+        async with httpx.AsyncClient(timeout=timeout) as client:
+            await client.get(setup_url)
+        return True
+    except httpx.HTTPError:
+        return False
+
+
 def get_customer_health(db: Session, customer_id: int) -> dict[str, Any]:
     """Check health of a customer's deployment."""
     deployment = db.query(Deployment).filter(Deployment.customer_id == customer_id).first()

app/services/scheduler_service.py

@@ -12,7 +12,14 @@ from datetime import datetime, timedelta
 
 from app.database import SessionLocal
 from app.models import Deployment, SystemConfig
-from app.services import image_service, netbird_client_update_service, update_log_service
+from app.services import (
+    docker_service,
+    image_service,
+    netbird_client_update_service,
+    netbird_service,
+    update_log_service,
+)
+from app.services.netbird_service import _log_action
 from app.utils.security import decrypt_value, encrypt_value
 
 logger = logging.getLogger(__name__)
@@ -55,6 +62,10 @@ async def _poll_loop() -> None:
             await _token_renewal_tick()
         except Exception:
             logger.exception("Token renewal tick failed")
+        try:
+            await _auto_restart_tick()
+        except Exception:
+            logger.exception("Auto-restart tick failed")
         await asyncio.sleep(_POLL_INTERVAL_SECONDS)
 
 
@@ -208,3 +219,65 @@ async def _run_check_and_optionally_apply(config: SystemConfig, apply_enabled: b
         )
     finally:
         log_db.close()
+
+
+async def _auto_restart_tick() -> None:
+    """Every poll interval, restart any customer whose NetBird dashboard
+    doesn't actually respond over HTTP(S) — a container can be "running" in
+    Docker while the web UI behind it is still unreachable, which is the
+    actual symptom customers hit, so this probes the real endpoint rather
+    than trusting container state alone. Falls back to container status
+    only if a customer has no setup_url yet. Guarded by
+    SystemConfig.auto_restart_unreachable_enabled and a per-deployment
+    cooldown so a persistently broken customer doesn't get restarted in a
+    tight loop, and skipped for deployments that aren't running/failed
+    (e.g. intentionally stopped or still pending).
+    """
+    db = SessionLocal()
+    try:
+        config = db.query(SystemConfig).filter(SystemConfig.id == 1).first()
+        if not config or not config.auto_restart_unreachable_enabled:
+            return
+
+        cooldown = timedelta(minutes=config.auto_restart_cooldown_minutes or 30)
+        now = datetime.now()
+
+        deployments = db.query(Deployment).all()
+        for dep in deployments:
+            if dep.deployment_status not in ("running", "failed"):
+                continue
+            if dep.last_auto_restart_at and now - dep.last_auto_restart_at < cooldown:
+                continue
+
+            reason = None
+            if dep.setup_url:
+                if not await netbird_service.check_dashboard_reachable(dep.setup_url):
+                    reason = f"dashboard ({dep.setup_url}) did not respond"
+            else:
+                containers = await docker_service.get_container_status_async(dep.container_prefix)
+                if not containers or not all(c["status"] == "running" for c in containers):
+                    not_running = [c["name"] for c in containers if c["status"] != "running"] or ["no containers found"]
+                    reason = f"container(s) not running: {', '.join(not_running)}"
+
+            if not reason:
+                continue
+
+            logger.warning(
+                "Auto-restart: customer %s unreachable (%s) — restarting.",
+                dep.container_prefix, reason,
+            )
+            _log_action(
+                db, dep.customer_id, "auto_restart", "info",
+                f"Automatic restart triggered: {reason}.",
+            )
+            try:
+                result = await netbird_service.restart_customer(db, dep.customer_id)
+                if not result["success"]:
+                    logger.error("Auto-restart failed for customer %s.", dep.container_prefix)
+            except Exception:
+                logger.exception("Auto-restart failed for customer %s.", dep.container_prefix)
+
+            dep.last_auto_restart_at = now
+            db.commit()
+    finally:
+        db.close()

app/utils/validators.py

@@ -165,6 +165,9 @@ class SystemConfigUpdate(BaseModel):
     # Master default for the NetBird client (peer) automatic-updates feature
     netbird_client_auto_update_version: Optional[str] = Field(None, max_length=50)
     netbird_client_auto_update_always: Optional[bool] = None
+    # Automatic restart of unreachable customer containers
+    auto_restart_unreachable_enabled: Optional[bool] = None
+    auto_restart_cooldown_minutes: Optional[int] = Field(None, ge=1, le=1440)
 
     @field_validator("auto_update_check_time")
     @classmethod

requirements.txt

@@ -5,24 +5,24 @@ sqlalchemy==2.0.23
 aiosqlite==0.19.0
 pydantic==2.5.2
 pydantic-settings==2.1.0
-python-jose[cryptography]==3.3.0
+python-jose[cryptography]==3.4.0
 passlib[bcrypt]==1.7.4
 bcrypt==4.1.2
-cryptography==41.0.7
-python-multipart==0.0.6
-httpx==0.26.0
-jinja2==3.1.2
+cryptography==50.0.0
+python-multipart==0.0.31
+httpx==0.28.1
+jinja2==3.1.6
 docker==7.0.0
 requests<2.32.0
 urllib3<2
 psutil==5.9.7
 pyyaml==6.0.1
-msal==1.28.0
+msal==1.39.0
 pyotp==2.9.0
 qrcode[pil]==7.4.2
 slowapi==0.1.9
 pywinrm>=0.4.3
 ldap3>=2.9.1
-pytest==7.4.3
-pytest-asyncio==0.23.2
-pytest-httpx==0.28.0
+pytest==9.0.3
+pytest-asyncio==1.4.0
+pytest-httpx==0.36.2

static/index.html

@@ -669,6 +669,24 @@
                                             </div>
                                         </form>
                                         <hr>
+                                        <h6 data-i18n="monitoring.autoRestartTitle">Automatic Restart of Unreachable Containers</h6>
+                                        <p class="text-muted small" data-i18n="monitoring.autoRestartHint">Some customers only regain access to their NetBird web UI after a container restart. When enabled, the scheduler periodically checks whether each customer's dashboard actually responds and restarts their containers if it doesn't, logging the action in the customer's log.</p>
+                                        <form id="settings-auto-restart-form">
+                                            <div class="form-check mb-3">
+                                                <input class="form-check-input" type="checkbox" id="cfg-auto-restart-enabled">
+                                                <label class="form-check-label" for="cfg-auto-restart-enabled" data-i18n="monitoring.autoRestartEnabled">Automatically restart unreachable customer containers</label>
+                                            </div>
+                                            <div class="row g-3 align-items-end">
+                                                <div class="col-md-3">
+                                                    <label class="form-label" data-i18n="monitoring.autoRestartCooldown">Cooldown between restarts (minutes)</label>
+                                                    <input type="number" class="form-control" id="cfg-auto-restart-cooldown" min="1" max="1440" value="30">
+                                                </div>
+                                            </div>
+                                            <div class="mt-3">
+                                                <button type="submit" class="btn btn-primary btn-sm"><i class="bi bi-save me-1"></i><span data-i18n="monitoring.saveAutoRestartSettings">Save Automation</span></button>
+                                            </div>
+                                        </form>
+                                        <hr>
                                         <h6 data-i18n="customer.nbuMasterTitle">NetBird Client Auto-Updates (all customers)</h6>
                                         <p class="text-muted small" data-i18n="customer.nbuMasterHint">Controls the "Automatic Updates" setting inside every customer's own NetBird dashboard (Settings &gt; Clients). Set the default here, then push it to all customers at once. Individual customers can still be overridden from their detail page.</p>
                                         <form id="settings-nbu-master-form">

static/js/app.js

@@ -1096,6 +1096,9 @@ async function loadSettings() {
             ? new Date(cfg.auto_update_last_run_at).toLocaleString()
             : t('monitoring.autoUpdateNever');
 
+        document.getElementById('cfg-auto-restart-enabled').checked = cfg.auto_restart_unreachable_enabled || false;
+        document.getElementById('cfg-auto-restart-cooldown').value = cfg.auto_restart_cooldown_minutes || 30;
+
         const nbuVersion = cfg.netbird_client_auto_update_version || 'disabled';
         const nbuIsCustom = nbuVersion !== 'disabled' && nbuVersion !== 'latest';
         document.getElementById('cfg-nbu-version-select').value = nbuIsCustom ? 'custom' : nbuVersion;
@@ -1236,6 +1239,20 @@ document.getElementById('settings-auto-update-form').addEventListener('submit',
     }
 });
 
+// Automatic restart of unreachable customer containers
+document.getElementById('settings-auto-restart-form').addEventListener('submit', async (e) => {
+    e.preventDefault();
+    try {
+        await api('PUT', '/settings/system', {
+            auto_restart_unreachable_enabled: document.getElementById('cfg-auto-restart-enabled').checked,
+            auto_restart_cooldown_minutes: parseInt(document.getElementById('cfg-auto-restart-cooldown').value, 10) || 30,
+        });
+        showSettingsAlert('success', t('messages.imageSettingsSaved'));
+    } catch (err) {
+        showSettingsAlert('danger', t('errors.failed', { error: err.message }));
+    }
+});
+
 function _readNbuMasterForm() {
     const select = document.getElementById('cfg-nbu-version-select').value;
     const version = select === 'custom' ? document.getElementById('cfg-nbu-custom-version').value.trim() : select;

static/lang/de.json

@@ -455,6 +455,11 @@
     "autoUpdateLastRun": "Letzte automatische Prüfung",
     "autoUpdateNever": "Noch nie ausgeführt",
     "saveAutoUpdateSettings": "Automatisierung speichern",
+    "autoRestartTitle": "Automatischer Neustart nicht erreichbarer Container",
+    "autoRestartHint": "Manche Kunden erhalten den Zugriff auf ihre NetBird-Weboberfläche erst nach einem Container-Neustart zurück. Wenn aktiviert, prüft der Scheduler regelmäßig, ob die Weboberfläche jedes Kunden tatsächlich antwortet, und startet die Container bei Bedarf neu — mit Protokollierung im Kunden-Log.",
+    "autoRestartEnabled": "Nicht erreichbare Kunden-Container automatisch neu starten",
+    "autoRestartCooldown": "Wartezeit zwischen Neustarts (Minuten)",
+    "saveAutoRestartSettings": "Automatisierung speichern",
     "updateLogTitle": "Update-Verlauf",
     "noUpdateLogs": "Noch keine Update-Läufe protokolliert.",
     "logTypeScheduled": "Automatisch",

static/lang/en.json

@@ -362,6 +362,11 @@
     "autoUpdateLastRun": "Last automatic check",
     "autoUpdateNever": "Never run",
     "saveAutoUpdateSettings": "Save Automation",
+    "autoRestartTitle": "Automatic Restart of Unreachable Containers",
+    "autoRestartHint": "Some customers only regain access to their NetBird web UI after a container restart. When enabled, the scheduler periodically checks whether each customer's dashboard actually responds and restarts their containers if it doesn't, logging the action in the customer's log.",
+    "autoRestartEnabled": "Automatically restart unreachable customer containers",
+    "autoRestartCooldown": "Cooldown between restarts (minutes)",
+    "saveAutoRestartSettings": "Save Automation",
     "updateLogTitle": "Update Log",
     "noUpdateLogs": "No update runs logged yet.",
     "logTypeScheduled": "Automatic",