chore(release): bump version to 1.6.1

Co-Authored-By: Claude Sonnet 5 <[email protected]>

app/database.py

@@ -45,6 +45,7 @@ def init_db() -> None:
         Deployment,
         DeploymentLog,
         SystemConfig,
+        UpdateRunLog,
         User,
     )
 
@@ -132,6 +133,10 @@ def _run_migrations() -> None:
         ("deployments", "netbird_api_token_renewed_at", "TEXT"),
         ("system_config", "netbird_client_auto_update_version", "TEXT DEFAULT 'disabled'"),
         ("system_config", "netbird_client_auto_update_always", "BOOLEAN DEFAULT 0"),
+        # Automatic restart of unreachable customer containers
+        ("system_config", "auto_restart_unreachable_enabled", "BOOLEAN DEFAULT 0"),
+        ("system_config", "auto_restart_cooldown_minutes", "INTEGER DEFAULT 30"),
+        ("deployments", "last_auto_restart_at", "TEXT"),
     ]
     for table, column, col_type in migrations:
         if not _has_column(table, column):

app/main.py

@@ -34,7 +34,7 @@ logger = logging.getLogger(__name__)
 app = FastAPI(
     title="NetBird MSP Appliance",
     description="Multi-tenant NetBird management platform for MSPs",
-    version="1.4.2",
+    version="1.6.1",
     docs_url="/api/docs",
     redoc_url="/api/redoc",
     openapi_url="/api/openapi.json",

app/models.py

@@ -1,5 +1,6 @@
 """SQLAlchemy ORM models for NetBird MSP Appliance."""
 
+import json
 from datetime import datetime
 from typing import Optional
 
@@ -95,6 +96,7 @@ class Deployment(Base):
     )
     deployed_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
     last_health_check: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
+    last_auto_restart_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
 
     __table_args__ = (
         CheckConstraint(
@@ -127,6 +129,9 @@ class Deployment(Base):
             "last_health_check": (
                 self.last_health_check.isoformat() if self.last_health_check else None
             ),
+            "last_auto_restart_at": (
+                self.last_auto_restart_at.isoformat() if self.last_auto_restart_at else None
+            ),
         }
 
 
@@ -211,6 +216,10 @@ class SystemConfig(Base):
     auto_update_apply_enabled: Mapped[bool] = mapped_column(Boolean, default=False)
     auto_update_last_run_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
 
+    # Automatic restart of customer containers when found unreachable/not running
+    auto_restart_unreachable_enabled: Mapped[bool] = mapped_column(Boolean, default=False)
+    auto_restart_cooldown_minutes: Mapped[int] = mapped_column(Integer, default=30)
+
     # Master default for the NetBird *client* (peer) automatic-updates feature
     # (Settings > Clients > Automatic Updates inside each customer's own
     # NetBird dashboard) — pushed to customers via the NetBird Management API.
@@ -277,6 +286,8 @@ class SystemConfig(Base):
             "auto_update_last_run_at": (
                 self.auto_update_last_run_at.isoformat() if self.auto_update_last_run_at else None
             ),
+            "auto_restart_unreachable_enabled": bool(self.auto_restart_unreachable_enabled),
+            "auto_restart_cooldown_minutes": self.auto_restart_cooldown_minutes or 30,
             "netbird_client_auto_update_version": self.netbird_client_auto_update_version or "disabled",
             "netbird_client_auto_update_always": bool(self.netbird_client_auto_update_always),
             "created_at": self.created_at.isoformat() if self.created_at else None,
@@ -321,6 +332,56 @@ class DeploymentLog(Base):
         }
 
 
+class UpdateRunLog(Base):
+    """Audit trail for NetBird Docker image update runs (pull + container recreate).
+
+    Written for both scheduled (automatic) and manual (button-triggered)
+    runs, so the actual outcome — which image pulls failed, which customer
+    container recreates failed — is visible in the UI instead of only in
+    container logs.
+    """
+
+    __tablename__ = "update_run_logs"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    run_type: Mapped[str] = mapped_column(String(20), nullable=False)  # "scheduled" | "manual"
+    trigger: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)  # username, or "scheduler"
+    started_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
+    finished_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
+    status: Mapped[str] = mapped_column(
+        String(20), default="running"
+    )  # running | no_update | success | partial | failed
+    any_update_available: Mapped[Optional[bool]] = mapped_column(Boolean, nullable=True)
+    pull_attempted: Mapped[bool] = mapped_column(Boolean, default=False)
+    pull_success: Mapped[Optional[bool]] = mapped_column(Boolean, nullable=True)
+    pull_details: Mapped[Optional[str]] = mapped_column(Text, nullable=True)  # JSON: {image: {success, error}}
+    apply_attempted: Mapped[bool] = mapped_column(Boolean, default=False)
+    customers_updated: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
+    customers_total: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
+    customer_details: Mapped[Optional[str]] = mapped_column(Text, nullable=True)  # JSON list
+    error: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+
+    def to_dict(self) -> dict:
+        """Serialize to dict, parsing the embedded JSON blobs."""
+        return {
+            "id": self.id,
+            "run_type": self.run_type,
+            "trigger": self.trigger,
+            "started_at": self.started_at.isoformat() if self.started_at else None,
+            "finished_at": self.finished_at.isoformat() if self.finished_at else None,
+            "status": self.status,
+            "any_update_available": self.any_update_available,
+            "pull_attempted": self.pull_attempted,
+            "pull_success": self.pull_success,
+            "pull_details": json.loads(self.pull_details) if self.pull_details else None,
+            "apply_attempted": self.apply_attempted,
+            "customers_updated": self.customers_updated,
+            "customers_total": self.customers_total,
+            "customer_details": json.loads(self.customer_details) if self.customer_details else None,
+            "error": self.error,
+        }
+
+
 class User(Base):
     """Admin user model."""
 

app/routers/deployments.py

@@ -9,7 +9,7 @@ from sqlalchemy.orm import Session
 from app.database import SessionLocal, get_db
 from app.dependencies import get_current_user
 from app.models import Customer, Deployment, SystemConfig, User
-from app.services import docker_service, image_service, netbird_client_update_service, netbird_service
+from app.services import docker_service, image_service, netbird_client_update_service, netbird_service, update_log_service
 from app.utils.security import decrypt_value, encrypt_value
 from app.utils.validators import NetbirdApiTokenPayload, NetbirdClientAutoUpdatePayload
 
@@ -246,9 +246,18 @@ async def update_customer_images(
             status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="System not configured."
         )
 
+    started_at = datetime.utcnow()
     instance_dir = f"{config.data_dir}/{customer.subdomain}"
     result = await image_service.update_customer_containers(instance_dir, deployment.container_prefix)
 
+    update_log_service.record_run(
+        db, run_type="manual", trigger=current_user.username, started_at=started_at,
+        apply_attempted=True,
+        customer_results=[{
+            "customer_name": customer.name, "success": result["success"], "error": result.get("error"),
+        }],
+    )
+
     if not result["success"]:
         raise HTTPException(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,

app/routers/monitoring.py

@@ -4,6 +4,7 @@ import asyncio
 import logging
 import platform
 import time
+from datetime import datetime
 from typing import Any
 
 import psutil
@@ -12,8 +13,8 @@ from sqlalchemy.orm import Session
 
 from app.database import SessionLocal, get_db
 from app.dependencies import get_current_user
-from app.models import Customer, Deployment, SystemConfig, User
-from app.services import docker_service, image_service, netbird_client_update_service
+from app.models import Customer, Deployment, SystemConfig, UpdateRunLog, User
+from app.services import docker_service, image_service, netbird_client_update_service, update_log_service
 from app.utils.security import decrypt_value
 from app.utils.validators import NetbirdClientAutoUpdatePayload
 
@@ -194,14 +195,25 @@ async def pull_all_netbird_images(
         config.netbird_dashboard_image,
     ]
 
+    username = current_user.username
+
     async def _pull_bg() -> None:
+        started_at = datetime.utcnow()
         bg_db = SessionLocal()
         try:
             cfg = bg_db.query(SystemConfig).filter(SystemConfig.id == 1).first()
-            if cfg:
-                await image_service.pull_all_images(cfg)
-        except Exception:
+            if not cfg:
+                return
+            pull_result = await image_service.pull_all_images(cfg)
+            update_log_service.record_run(
+                bg_db, run_type="manual", trigger=username, started_at=started_at,
+                any_update_available=True, pull_attempted=True, pull_results=pull_result["results"],
+            )
+        except Exception as exc:
             logger.exception("Background image pull failed")
+            update_log_service.record_run(
+                bg_db, run_type="manual", trigger=username, started_at=started_at, error=str(exc),
+            )
         finally:
             bg_db.close()
 
@@ -285,6 +297,28 @@ async def apply_netbird_client_updates_to_all(
     }
 
 
+@router.get("/update-logs")
+async def get_update_logs(
+    limit: int = 30,
+    current_user: User = Depends(get_current_user),
+    db: Session = Depends(get_db),
+) -> list[dict[str, Any]]:
+    """Return the most recent NetBird image update runs (scheduled and manual).
+
+    Shows exactly what happened on each run: whether an image pull was
+    attempted and its per-image result, and whether customer containers
+    were recreated and which ones failed — so this doesn't have to be
+    reconstructed from container logs.
+    """
+    logs = (
+        db.query(UpdateRunLog)
+        .order_by(UpdateRunLog.started_at.desc())
+        .limit(min(limit, 100))
+        .all()
+    )
+    return [log.to_dict() for log in logs]
+
+
 @router.post("/customers/update-all")
 async def update_all_customers(
     current_user: User = Depends(get_current_user),
@@ -299,6 +333,7 @@ async def update_all_customers(
     if current_user.role != "admin":
         raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin only.")
 
+    started_at = datetime.utcnow()
     config = db.query(SystemConfig).filter(SystemConfig.id == 1).first()
     if not config:
         raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="System not configured.")
@@ -317,6 +352,10 @@ async def update_all_customers(
             })
 
     if not to_update:
+        update_log_service.record_run(
+            db, run_type="manual", trigger=current_user.username,
+            started_at=started_at, apply_attempted=True, customer_results=[],
+        )
         return {"message": "All customers are already up to date.", "updated": 0, "results": []}
 
     # Update customers sequentially — one at a time. A failure for one
@@ -343,6 +382,10 @@ async def update_all_customers(
         })
 
     success_count = sum(1 for r in update_results if r["success"])
+    update_log_service.record_run(
+        db, run_type="manual", trigger=current_user.username,
+        started_at=started_at, apply_attempted=True, customer_results=update_results,
+    )
     return {
         "message": f"Updated {success_count} of {len(update_results)} customer(s).",
         "updated": success_count,

app/services/netbird_service.py

@@ -26,6 +26,7 @@ import urllib.error
 from datetime import datetime
 from typing import Any
 
+import httpx
 from jinja2 import Environment, FileSystemLoader
 from sqlalchemy.orm import Session
 
@@ -529,6 +530,25 @@ async def restart_customer(db: Session, customer_id: int) -> dict[str, Any]:
     return {"success": ok}
 
 
+async def check_dashboard_reachable(setup_url: str, timeout: float = 10) -> bool:
+    """Check whether a customer's NetBird dashboard actually responds over HTTP(S).
+
+    A container can be "running" in Docker while the web UI behind it is
+    still unreachable (e.g. the management service is up but not accepting
+    connections yet), so this checks the real endpoint rather than the
+    container state. Any HTTP response (including 4xx/5xx from the app
+    itself) counts as reachable — only connection failures/timeouts don't.
+    """
+    if not setup_url:
+        return True  # nothing to check (e.g. deployment not fully set up yet)
+    try:
+        async with httpx.AsyncClient(timeout=timeout) as client:
+            await client.get(setup_url)
+        return True
+    except httpx.HTTPError:
+        return False
+
+
 def get_customer_health(db: Session, customer_id: int) -> dict[str, Any]:
     """Check health of a customer's deployment."""
     deployment = db.query(Deployment).filter(Deployment.customer_id == customer_id).first()

app/services/scheduler_service.py

@@ -12,7 +12,14 @@ from datetime import datetime, timedelta
 
 from app.database import SessionLocal
 from app.models import Deployment, SystemConfig
-from app.services import image_service, netbird_client_update_service
+from app.services import (
+    docker_service,
+    image_service,
+    netbird_client_update_service,
+    netbird_service,
+    update_log_service,
+)
+from app.services.netbird_service import _log_action
 from app.utils.security import decrypt_value, encrypt_value
 
 logger = logging.getLogger(__name__)
@@ -55,6 +62,10 @@ async def _poll_loop() -> None:
             await _token_renewal_tick()
         except Exception:
             logger.exception("Token renewal tick failed")
+        try:
+            await _auto_restart_tick()
+        except Exception:
+            logger.exception("Auto-restart tick failed")
         await asyncio.sleep(_POLL_INTERVAL_SECONDS)
 
 
@@ -125,29 +136,49 @@ async def _tick() -> None:
         logger.info(
             "Running scheduled NetBird image update check (auto-apply=%s)...", apply_enabled
         )
-        await _run_check_and_optionally_apply(config, apply_enabled)
+        await _run_check_and_optionally_apply(config, apply_enabled, started_at=now)
     finally:
         db.close()
 
 
-async def _run_check_and_optionally_apply(config: SystemConfig, apply_enabled: bool) -> None:
-    hub_status = await image_service.check_all_images(config)
-    if not hub_status["any_update_available"]:
-        logger.info("Scheduled check: all NetBird images already up to date.")
-        return
+async def _run_check_and_optionally_apply(config: SystemConfig, apply_enabled: bool, started_at: datetime) -> None:
+    """Check Docker Hub, pull if needed, then (if enabled) recreate any
+    customer container that isn't running the locally-pulled image yet.
 
-    logger.info("Scheduled check: new NetBird image(s) available — pulling.")
-    pull_result = await image_service.pull_all_images(config)
-    if not pull_result["all_success"]:
-        logger.error("Scheduled image pull had failures: %s", pull_result["results"])
-
-    if not apply_enabled:
-        logger.info("Auto-apply disabled — images pulled, customer containers left untouched.")
-        return
-
-    db = SessionLocal()
+    The "recreate" step is intentionally NOT gated on a Hub update having
+    just been found: a customer can already be behind the *locally pulled*
+    image from an earlier check/pull that was never applied (e.g. auto-apply
+    was off at the time, or a previous run failed) — that backlog must keep
+    being cleared on every run, not just on runs that also found something
+    new on Hub, otherwise it silently never happens.
+    """
+    log_db = SessionLocal()
     try:
-        deployments = db.query(Deployment).all()
+        hub_status = await image_service.check_all_images(config)
+        any_update = hub_status["any_update_available"]
+
+        pull_attempted = False
+        pull_results = None
+        if any_update:
+            logger.info("Scheduled check: new NetBird image(s) available — pulling.")
+            pull_attempted = True
+            pull_result = await image_service.pull_all_images(config)
+            pull_results = pull_result["results"]
+            if not pull_result["all_success"]:
+                logger.error("Scheduled image pull had failures: %s", pull_results)
+        else:
+            logger.info("Scheduled check: all NetBird images already up to date on Docker Hub.")
+
+        if not apply_enabled:
+            logger.info("Auto-apply disabled — customer containers left untouched.")
+            update_log_service.record_run(
+                log_db, run_type="scheduled", trigger="scheduler",
+                started_at=started_at, any_update_available=any_update,
+                pull_attempted=pull_attempted, pull_results=pull_results,
+            )
+            return
+
+        deployments = log_db.query(Deployment).all()
         to_update = []
         for dep in deployments:
             cs = image_service.get_customer_container_image_status(dep.container_prefix, config)
@@ -159,16 +190,94 @@ async def _run_check_and_optionally_apply(config: SystemConfig, apply_enabled: b
                     "customer_name": customer.name,
                 })
         logger.info("Scheduled auto-apply: updating %d customer(s)...", len(to_update))
+        customer_results = []
         for entry in to_update:
             try:
                 res = await image_service.update_customer_containers(
                     entry["instance_dir"], entry["project_name"]
                 )
-                logger.info(
-                    "Scheduled update for %s: %s",
-                    entry["customer_name"], "OK" if res["success"] else res.get("error"),
-                )
-            except Exception:
+                ok = res["success"]
+                err = res.get("error")
+            except Exception as exc:
                 logger.exception("Scheduled update failed for %s", entry["customer_name"])
+                ok = False
+                err = str(exc)
+            logger.info("Scheduled update for %s: %s", entry["customer_name"], "OK" if ok else err)
+            customer_results.append({"customer_name": entry["customer_name"], "success": ok, "error": err})
+
+        update_log_service.record_run(
+            log_db, run_type="scheduled", trigger="scheduler",
+            started_at=started_at, any_update_available=any_update,
+            pull_attempted=pull_attempted, pull_results=pull_results,
+            apply_attempted=True, customer_results=customer_results,
+        )
+    except Exception as exc:
+        logger.exception("Scheduled update run failed")
+        update_log_service.record_run(
+            log_db, run_type="scheduled", trigger="scheduler",
+            started_at=started_at, error=str(exc),
+        )
+    finally:
+        log_db.close()
+
+
+async def _auto_restart_tick() -> None:
+    """Every poll interval, restart any customer whose NetBird dashboard
+    doesn't actually respond over HTTP(S) — a container can be "running" in
+    Docker while the web UI behind it is still unreachable, which is the
+    actual symptom customers hit, so this probes the real endpoint rather
+    than trusting container state alone. Falls back to container status
+    only if a customer has no setup_url yet. Guarded by
+    SystemConfig.auto_restart_unreachable_enabled and a per-deployment
+    cooldown so a persistently broken customer doesn't get restarted in a
+    tight loop, and skipped for deployments that aren't running/failed
+    (e.g. intentionally stopped or still pending).
+    """
+    db = SessionLocal()
+    try:
+        config = db.query(SystemConfig).filter(SystemConfig.id == 1).first()
+        if not config or not config.auto_restart_unreachable_enabled:
+            return
+
+        cooldown = timedelta(minutes=config.auto_restart_cooldown_minutes or 30)
+        now = datetime.now()
+
+        deployments = db.query(Deployment).all()
+        for dep in deployments:
+            if dep.deployment_status not in ("running", "failed"):
+                continue
+            if dep.last_auto_restart_at and now - dep.last_auto_restart_at < cooldown:
+                continue
+
+            reason = None
+            if dep.setup_url:
+                if not await netbird_service.check_dashboard_reachable(dep.setup_url):
+                    reason = f"dashboard ({dep.setup_url}) did not respond"
+            else:
+                containers = await docker_service.get_container_status_async(dep.container_prefix)
+                if not containers or not all(c["status"] == "running" for c in containers):
+                    not_running = [c["name"] for c in containers if c["status"] != "running"] or ["no containers found"]
+                    reason = f"container(s) not running: {', '.join(not_running)}"
+
+            if not reason:
+                continue
+
+            logger.warning(
+                "Auto-restart: customer %s unreachable (%s) — restarting.",
+                dep.container_prefix, reason,
+            )
+            _log_action(
+                db, dep.customer_id, "auto_restart", "info",
+                f"Automatic restart triggered: {reason}.",
+            )
+            try:
+                result = await netbird_service.restart_customer(db, dep.customer_id)
+                if not result["success"]:
+                    logger.error("Auto-restart failed for customer %s.", dep.container_prefix)
+            except Exception:
+                logger.exception("Auto-restart failed for customer %s.", dep.container_prefix)
+
+            dep.last_auto_restart_at = now
+            db.commit()
     finally:
         db.close()

app/services/update_log_service.py

@@ -0,0 +1,84 @@
+"""Persists the outcome of every NetBird Docker image update run.
+
+Both the automatic scheduler and the manual "Check/Pull/Update" buttons in
+the UI write here, so it's visible afterwards which run pulled which image
+successfully, and which customer container recreate failed — instead of
+that information only existing in container logs.
+"""
+
+import json
+import logging
+from datetime import datetime
+from typing import Any, Optional
+
+from sqlalchemy.orm import Session
+
+from app.models import UpdateRunLog
+
+logger = logging.getLogger(__name__)
+
+
+def record_run(
+    db: Session,
+    *,
+    run_type: str,
+    trigger: Optional[str],
+    started_at: datetime,
+    any_update_available: Optional[bool] = None,
+    pull_attempted: bool = False,
+    pull_results: Optional[dict[str, Any]] = None,
+    apply_attempted: bool = False,
+    customer_results: Optional[list[dict[str, Any]]] = None,
+    error: Optional[str] = None,
+) -> UpdateRunLog:
+    """Write one completed update-run entry.
+
+    Args:
+        run_type: "scheduled" or "manual".
+        trigger: username for manual runs, "scheduler" for automatic ones.
+        pull_results: {image: {"success": bool, "error": str|None}} as returned
+            by image_service.pull_all_images()["results"].
+        customer_results: list of {"customer_name", "success", "error"} as
+            returned by the update-all / update-customer flows.
+    """
+    pull_success = None
+    if pull_results:
+        pull_success = all(r.get("success") for r in pull_results.values())
+
+    customers_total = len(customer_results) if customer_results is not None else None
+    customers_updated = (
+        sum(1 for r in customer_results if r.get("success")) if customer_results is not None else None
+    )
+
+    if error:
+        status = "failed"
+    elif pull_attempted and pull_success is False:
+        status = "failed"
+    elif apply_attempted and customers_total and customers_updated != customers_total:
+        status = "partial"
+    elif not pull_attempted and not customers_total:
+        # Nothing to pull (Hub already matched local) and no customer
+        # container was behind the locally pulled image either.
+        status = "no_update"
+    else:
+        status = "success"
+
+    log = UpdateRunLog(
+        run_type=run_type,
+        trigger=trigger,
+        started_at=started_at,
+        finished_at=datetime.utcnow(),
+        status=status,
+        any_update_available=any_update_available,
+        pull_attempted=pull_attempted,
+        pull_success=pull_success,
+        pull_details=json.dumps(pull_results) if pull_results is not None else None,
+        apply_attempted=apply_attempted,
+        customers_updated=customers_updated,
+        customers_total=customers_total,
+        customer_details=json.dumps(customer_results) if customer_results is not None else None,
+        error=error,
+    )
+    db.add(log)
+    db.commit()
+    return log

app/utils/validators.py

@@ -165,6 +165,9 @@ class SystemConfigUpdate(BaseModel):
     # Master default for the NetBird client (peer) automatic-updates feature
     netbird_client_auto_update_version: Optional[str] = Field(None, max_length=50)
     netbird_client_auto_update_always: Optional[bool] = None
+    # Automatic restart of unreachable customer containers
+    auto_restart_unreachable_enabled: Optional[bool] = None
+    auto_restart_cooldown_minutes: Optional[int] = Field(None, ge=1, le=1440)
 
     @field_validator("auto_update_check_time")
     @classmethod

requirements.txt

@@ -5,24 +5,24 @@ sqlalchemy==2.0.23
 aiosqlite==0.19.0
 pydantic==2.5.2
 pydantic-settings==2.1.0
-python-jose[cryptography]==3.3.0
+python-jose[cryptography]==3.4.0
 passlib[bcrypt]==1.7.4
 bcrypt==4.1.2
-cryptography==41.0.7
-python-multipart==0.0.6
-httpx==0.26.0
-jinja2==3.1.2
+cryptography==50.0.0
+python-multipart==0.0.31
+httpx==0.28.1
+jinja2==3.1.6
 docker==7.0.0
 requests<2.32.0
 urllib3<2
 psutil==5.9.7
 pyyaml==6.0.1
-msal==1.28.0
+msal==1.39.0
 pyotp==2.9.0
 qrcode[pil]==7.4.2
 slowapi==0.1.9
 pywinrm>=0.4.3
 ldap3>=2.9.1
-pytest==7.4.3
-pytest-asyncio==0.23.2
-pytest-httpx==0.28.0
+pytest==9.0.3
+pytest-asyncio==1.4.0
+pytest-httpx==0.36.2

static/index.html

@@ -669,6 +669,24 @@
                                             </div>
                                         </form>
                                         <hr>
+                                        <h6 data-i18n="monitoring.autoRestartTitle">Automatic Restart of Unreachable Containers</h6>
+                                        <p class="text-muted small" data-i18n="monitoring.autoRestartHint">Some customers only regain access to their NetBird web UI after a container restart. When enabled, the scheduler periodically checks whether each customer's dashboard actually responds and restarts their containers if it doesn't, logging the action in the customer's log.</p>
+                                        <form id="settings-auto-restart-form">
+                                            <div class="form-check mb-3">
+                                                <input class="form-check-input" type="checkbox" id="cfg-auto-restart-enabled">
+                                                <label class="form-check-label" for="cfg-auto-restart-enabled" data-i18n="monitoring.autoRestartEnabled">Automatically restart unreachable customer containers</label>
+                                            </div>
+                                            <div class="row g-3 align-items-end">
+                                                <div class="col-md-3">
+                                                    <label class="form-label" data-i18n="monitoring.autoRestartCooldown">Cooldown between restarts (minutes)</label>
+                                                    <input type="number" class="form-control" id="cfg-auto-restart-cooldown" min="1" max="1440" value="30">
+                                                </div>
+                                            </div>
+                                            <div class="mt-3">
+                                                <button type="submit" class="btn btn-primary btn-sm"><i class="bi bi-save me-1"></i><span data-i18n="monitoring.saveAutoRestartSettings">Save Automation</span></button>
+                                            </div>
+                                        </form>
+                                        <hr>
                                         <h6 data-i18n="customer.nbuMasterTitle">NetBird Client Auto-Updates (all customers)</h6>
                                         <p class="text-muted small" data-i18n="customer.nbuMasterHint">Controls the "Automatic Updates" setting inside every customer's own NetBird dashboard (Settings &gt; Clients). Set the default here, then push it to all customers at once. Individual customers can still be overridden from their detail page.</p>
                                         <form id="settings-nbu-master-form">
@@ -1260,6 +1278,19 @@
                         <p class="text-muted mb-0" data-i18n="monitoring.clickCheckUpdates">Click "Check for Updates" to compare local images with Docker Hub.</p>
                     </div>
                 </div>
+
+                <!-- Update Run Log -->
+                <div class="card shadow-sm mt-4">
+                    <div class="card-header d-flex justify-content-between align-items-center">
+                        <span><i class="bi bi-clock-history me-2"></i><span data-i18n="monitoring.updateLogTitle">Update Log</span></span>
+                        <button class="btn btn-outline-secondary btn-sm" onclick="loadUpdateLog()" id="btn-refresh-update-log">
+                            <i class="bi bi-arrow-clockwise me-1"></i><span data-i18n="monitoring.refresh">Refresh</span>
+                        </button>
+                    </div>
+                    <div class="card-body" id="update-log-body">
+                        <p class="text-muted mb-0" data-i18n="common.loading">Loading...</p>
+                    </div>
+                </div>
             </div>
         </div>
     </div>

static/js/app.js

@@ -1096,6 +1096,9 @@ async function loadSettings() {
             ? new Date(cfg.auto_update_last_run_at).toLocaleString()
             : t('monitoring.autoUpdateNever');
 
+        document.getElementById('cfg-auto-restart-enabled').checked = cfg.auto_restart_unreachable_enabled || false;
+        document.getElementById('cfg-auto-restart-cooldown').value = cfg.auto_restart_cooldown_minutes || 30;
+
         const nbuVersion = cfg.netbird_client_auto_update_version || 'disabled';
         const nbuIsCustom = nbuVersion !== 'disabled' && nbuVersion !== 'latest';
         document.getElementById('cfg-nbu-version-select').value = nbuIsCustom ? 'custom' : nbuVersion;
@@ -1236,6 +1239,20 @@ document.getElementById('settings-auto-update-form').addEventListener('submit',
     }
 });
 
+// Automatic restart of unreachable customer containers
+document.getElementById('settings-auto-restart-form').addEventListener('submit', async (e) => {
+    e.preventDefault();
+    try {
+        await api('PUT', '/settings/system', {
+            auto_restart_unreachable_enabled: document.getElementById('cfg-auto-restart-enabled').checked,
+            auto_restart_cooldown_minutes: parseInt(document.getElementById('cfg-auto-restart-cooldown').value, 10) || 30,
+        });
+        showSettingsAlert('success', t('messages.imageSettingsSaved'));
+    } catch (err) {
+        showSettingsAlert('danger', t('errors.failed', { error: err.message }));
+    }
+});
+
 function _readNbuMasterForm() {
     const select = document.getElementById('cfg-nbu-version-select').value;
     const version = select === 'custom' ? document.getElementById('cfg-nbu-custom-version').value.trim() : select;
@@ -1943,7 +1960,90 @@ function togglePasswordVisibility(inputId) {
 // Monitoring
 // ---------------------------------------------------------------------------
 async function loadMonitoring() {
-    await Promise.all([loadResources(), loadAllCustomerStatuses()]);
+    await Promise.all([loadResources(), loadAllCustomerStatuses(), loadUpdateLog()]);
+}
+
+const _updateLogStatusBadge = {
+    success: 'bg-success',
+    no_update: 'bg-secondary',
+    partial: 'bg-warning text-dark',
+    failed: 'bg-danger',
+    running: 'bg-info',
+};
+
+async function loadUpdateLog() {
+    const body = document.getElementById('update-log-body');
+    body.innerHTML = `<span class="spinner-border spinner-border-sm me-2"></span>${t('common.loading')}`;
+    try {
+        const logs = await api('GET', '/monitoring/update-logs?limit=30');
+        if (logs.length === 0) {
+            body.innerHTML = `<p class="text-muted mb-0">${t('monitoring.noUpdateLogs')}</p>`;
+            return;
+        }
+        const rows = logs.map((l, i) => {
+            const badgeClass = _updateLogStatusBadge[l.status] || 'bg-secondary';
+            const typeLabel = l.run_type === 'scheduled' ? t('monitoring.logTypeScheduled') : t('monitoring.logTypeManual');
+            const pullSummary = l.pull_attempted
+                ? (l.pull_success ? t('monitoring.logPullOk') : t('monitoring.logPullFailed'))
+                : t('monitoring.logPullSkipped');
+            const applySummary = l.apply_attempted
+                ? t('monitoring.logApplySummary', { updated: l.customers_updated ?? 0, total: l.customers_total ?? 0 })
+                : t('monitoring.logApplySkipped');
+            const detailId = `update-log-detail-${i}`;
+            const hasDetails = (l.pull_details && Object.keys(l.pull_details).length > 0) ||
+                (l.customer_details && l.customer_details.length > 0) || l.error;
+            const detailsHtml = hasDetails ? `
+                <div class="collapse mt-2" id="${detailId}">
+                    ${l.error ? `<div class="alert alert-danger py-2 small mb-2">${esc(l.error)}</div>` : ''}
+                    ${l.pull_details ? `
+                        <table class="table table-sm mb-2">
+                            <thead><tr><th>${t('monitoring.thImage')}</th><th>${t('monitoring.thStatus')}</th><th></th></tr></thead>
+                            <tbody>${Object.entries(l.pull_details).map(([img, r]) => `<tr>
+                                <td><code class="small">${esc(img)}</code></td>
+                                <td>${r.success ? `<span class="badge bg-success">OK</span>` : `<span class="badge bg-danger">Error</span>`}</td>
+                                <td class="small text-muted">${esc(r.error || '')}</td>
+                            </tr>`).join('')}</tbody>
+                        </table>` : ''}
+                    ${l.customer_details && l.customer_details.length > 0 ? `
+                        <table class="table table-sm mb-0">
+                            <thead><tr><th>${t('monitoring.thName')}</th><th>${t('monitoring.thStatus')}</th><th></th></tr></thead>
+                            <tbody>${l.customer_details.map(r => `<tr>
+                                <td>${esc(r.customer_name)}</td>
+                                <td>${r.success ? `<span class="badge bg-success">OK</span>` : `<span class="badge bg-danger">Error</span>`}</td>
+                                <td class="small text-muted">${esc(r.error || '')}</td>
+                            </tr>`).join('')}</tbody>
+                        </table>` : ''}
+                </div>` : '';
+            return `<tr>
+                <td class="small text-nowrap">${formatDate(l.started_at)}</td>
+                <td><span class="badge bg-light text-dark border">${typeLabel}</span></td>
+                <td><span class="badge ${badgeClass}">${t('monitoring.logStatus_' + l.status)}</span></td>
+                <td class="small">${pullSummary}</td>
+                <td class="small">${applySummary}</td>
+                <td class="small text-muted">${esc(l.trigger || '')}</td>
+                <td>${hasDetails ? `<button class="btn btn-sm btn-outline-secondary" data-bs-toggle="collapse" data-bs-target="#${detailId}"><i class="bi bi-chevron-down"></i></button>` : ''}</td>
+            </tr>${detailsHtml ? `<tr class="d-none"></tr><tr><td colspan="7" class="p-0 border-0">${detailsHtml}</td></tr>` : ''}`;
+        }).join('');
+        body.innerHTML = `
+            <div class="table-responsive">
+                <table class="table table-sm mb-0">
+                    <thead class="table-light">
+                        <tr>
+                            <th>${t('monitoring.logTime')}</th>
+                            <th>${t('monitoring.logType')}</th>
+                            <th>${t('monitoring.thStatus')}</th>
+                            <th>${t('monitoring.logPull')}</th>
+                            <th>${t('monitoring.logApply')}</th>
+                            <th>${t('monitoring.logTrigger')}</th>
+                            <th></th>
+                        </tr>
+                    </thead>
+                    <tbody>${rows}</tbody>
+                </table>
+            </div>`;
+    } catch (err) {
+        body.innerHTML = `<div class="alert alert-danger">${esc(err.message)}</div>`;
+    }
 }
 
 async function loadResources() {

static/lang/de.json

@@ -454,6 +454,30 @@
     "autoUpdateApplyHint": "Wenn aktiviert, werden nach einer gefundenen Aktualisierung automatisch alle Kunden-Container neu erstellt (kurzer Neustart der Dienste). Wenn deaktiviert, werden nur die Images geladen — die Aktualisierung der Kunden erfolgt weiterhin manuell.",
     "autoUpdateLastRun": "Letzte automatische Prüfung",
     "autoUpdateNever": "Noch nie ausgeführt",
-    "saveAutoUpdateSettings": "Automatisierung speichern"
+    "saveAutoUpdateSettings": "Automatisierung speichern",
+    "autoRestartTitle": "Automatischer Neustart nicht erreichbarer Container",
+    "autoRestartHint": "Manche Kunden erhalten den Zugriff auf ihre NetBird-Weboberfläche erst nach einem Container-Neustart zurück. Wenn aktiviert, prüft der Scheduler regelmäßig, ob die Weboberfläche jedes Kunden tatsächlich antwortet, und startet die Container bei Bedarf neu — mit Protokollierung im Kunden-Log.",
+    "autoRestartEnabled": "Nicht erreichbare Kunden-Container automatisch neu starten",
+    "autoRestartCooldown": "Wartezeit zwischen Neustarts (Minuten)",
+    "saveAutoRestartSettings": "Automatisierung speichern",
+    "updateLogTitle": "Update-Verlauf",
+    "noUpdateLogs": "Noch keine Update-Läufe protokolliert.",
+    "logTypeScheduled": "Automatisch",
+    "logTypeManual": "Manuell",
+    "logTime": "Zeitpunkt",
+    "logType": "Typ",
+    "logPull": "Image-Download",
+    "logApply": "Kunden-Update",
+    "logTrigger": "Ausgelöst von",
+    "logPullOk": "Erfolgreich",
+    "logPullFailed": "Fehlgeschlagen",
+    "logPullSkipped": "Nicht nötig",
+    "logApplySummary": "{updated}/{total} Kunden",
+    "logApplySkipped": "Nicht ausgeführt",
+    "logStatus_success": "Erfolgreich",
+    "logStatus_no_update": "Kein Update nötig",
+    "logStatus_partial": "Teilweise",
+    "logStatus_failed": "Fehlgeschlagen",
+    "logStatus_running": "Läuft…"
   }
 }

static/lang/en.json

@@ -361,7 +361,31 @@
     "autoUpdateApplyHint": "When enabled, all customer containers are automatically recreated after a new image is found (services briefly restart). When disabled, only the images are pulled — updating customers stays a manual step.",
     "autoUpdateLastRun": "Last automatic check",
     "autoUpdateNever": "Never run",
-    "saveAutoUpdateSettings": "Save Automation"
+    "saveAutoUpdateSettings": "Save Automation",
+    "autoRestartTitle": "Automatic Restart of Unreachable Containers",
+    "autoRestartHint": "Some customers only regain access to their NetBird web UI after a container restart. When enabled, the scheduler periodically checks whether each customer's dashboard actually responds and restarts their containers if it doesn't, logging the action in the customer's log.",
+    "autoRestartEnabled": "Automatically restart unreachable customer containers",
+    "autoRestartCooldown": "Cooldown between restarts (minutes)",
+    "saveAutoRestartSettings": "Save Automation",
+    "updateLogTitle": "Update Log",
+    "noUpdateLogs": "No update runs logged yet.",
+    "logTypeScheduled": "Automatic",
+    "logTypeManual": "Manual",
+    "logTime": "Time",
+    "logType": "Type",
+    "logPull": "Image Pull",
+    "logApply": "Customer Update",
+    "logTrigger": "Triggered by",
+    "logPullOk": "Succeeded",
+    "logPullFailed": "Failed",
+    "logPullSkipped": "Not needed",
+    "logApplySummary": "{updated}/{total} customers",
+    "logApplySkipped": "Not run",
+    "logStatus_success": "Success",
+    "logStatus_no_update": "No update needed",
+    "logStatus_partial": "Partial",
+    "logStatus_failed": "Failed",
+    "logStatus_running": "Running…"
   },
   "userModal": {
     "title": "New User",