chore(release): bump version to 1.6.0

Co-Authored-By: Claude Sonnet 5 <[email protected]>

README.md

@@ -23,6 +23,7 @@ A management solution for running isolated NetBird instances for your MSP busine
 - [Troubleshooting](#troubleshooting)
 - [Updates](#updates)
 - [Security Best Practices](#security-best-practices)
+- [Performance Tuning](#performance-tuning)
 - [License](#license)
 
 ---
@@ -44,6 +45,7 @@ A management solution for running isolated NetBird instances for your MSP busine
 - **Start / Stop / Restart** — Control customer instances from the dashboard
 - **Customer Status Tracking** — Automatic status sync (active / inactive / error)
 - **Update Indicators** — Per-customer badges when container images are outdated
+- **Sortable Columns** — Click any customer list column header (ID, Name, Subdomain, Status, Devices, Created) to sort ascending/descending
 
 ### NetBird Container Updates
 - **Docker Hub Digest Check** — Compare locally pulled image digests against Docker Hub without pulling
@@ -74,7 +76,7 @@ A management solution for running isolated NetBird instances for your MSP busine
 
 ### Integrations
 - **Windows DNS** — Automatically create and delete DNS A-records when deploying or removing customers
-- **MSP Updates** — In-UI appliance update check with configurable release branch
+- **MSP Updates** — In-UI appliance version check, configurable release branch, and one-click background update with live progress
 
 ---
 
@@ -483,7 +485,7 @@ http://your-server:8000/api/docs
 **Common Endpoints:**
 ```
 POST   /api/customers                    # Create customer + deploy
-GET    /api/customers                    # List all customers
+GET    /api/customers                    # List customers (supports search, status filter, sort_by/sort_order)
 GET    /api/customers/{id}               # Get customer details
 PUT    /api/customers/{id}               # Update customer
 DELETE /api/customers/{id}               # Delete customer
@@ -498,6 +500,9 @@ POST   /api/customers/{id}/update-images # Recreate containers with new images
 GET    /api/settings/branding            # Get branding (public, no auth)
 GET    /api/settings/npm-certificates    # List NPM SSL certificates
 PUT    /api/settings                     # Update system settings
+GET    /api/settings/version             # Current + latest available appliance version
+POST   /api/settings/update              # Start appliance update in the background
+GET    /api/settings/update/status       # Poll update progress (backup/pull/build/restart)
 
 GET    /api/users                        # List users
 POST   /api/users                        # Create user
@@ -581,6 +586,15 @@ docker logs -f netbird-msp-appliance
 
 ### Updating the Appliance
 
+The recommended way to update is the built-in one-click updater:
+
+1. Go to **Settings > NetBird MSP Updates**
+2. Configure the Git repository URL and branch (defaults to `main`) if not already set
+3. Click **"Update starten"** ("Start Update")
+
+This backs up the database, pulls the configured branch, rebuilds the container image, and swaps in the new container — all in the background. The page shows live progress (backup → pull → build → restart) and automatically detects when the app is back up, so there's no need to babysit a terminal. The app is unavailable for roughly 30-60 seconds during the container swap.
+
+**Manual update (fallback, e.g. no Web UI access):**
 ```bash
 cd /opt/netbird-msp
 git pull
@@ -588,7 +602,7 @@ docker compose down
 docker compose up -d --build
 ```
 
-The database migrations run automatically on startup.
+The database migrations run automatically on startup either way.
 
 ### Updating NetBird Images
 
@@ -667,7 +681,7 @@ MIT License — see [LICENSE](LICENSE) file for details.
 
 ## Built With AI
 
-This software was developed with [Claude Code](https://claude.ai/claude-code) (Anthropic Claude Sonnet 4.6) — from architecture and backend logic to frontend UI and deployment scripts.
+This software was developed and is continuously maintained with [Claude Code](https://claude.ai/claude-code) (Anthropic) — from architecture and backend logic to frontend UI and deployment scripts.
 
 ## Acknowledgments
 

app/database.py

@@ -45,6 +45,7 @@ def init_db() -> None:
         Deployment,
         DeploymentLog,
         SystemConfig,
+        UpdateRunLog,
         User,
     )
 
@@ -122,6 +123,20 @@ def _run_migrations() -> None:
         ("system_config", "git_repo_url", "TEXT"),
         ("system_config", "git_branch", "TEXT DEFAULT 'main'"),
         ("system_config", "git_token_encrypted", "TEXT"),
+        # Automatic NetBird image update check/apply
+        ("system_config", "auto_update_check_enabled", "BOOLEAN DEFAULT 0"),
+        ("system_config", "auto_update_check_time", "TEXT DEFAULT '03:00'"),
+        ("system_config", "auto_update_apply_enabled", "BOOLEAN DEFAULT 0"),
+        ("system_config", "auto_update_last_run_at", "TEXT"),
+        # NetBird client (peer) automatic-updates master default + per-customer PAT
+        ("deployments", "netbird_api_token_encrypted", "TEXT"),
+        ("deployments", "netbird_api_token_renewed_at", "TEXT"),
+        ("system_config", "netbird_client_auto_update_version", "TEXT DEFAULT 'disabled'"),
+        ("system_config", "netbird_client_auto_update_always", "BOOLEAN DEFAULT 0"),
+        # Automatic restart of unreachable customer containers
+        ("system_config", "auto_restart_unreachable_enabled", "BOOLEAN DEFAULT 0"),
+        ("system_config", "auto_restart_cooldown_minutes", "INTEGER DEFAULT 30"),
+        ("deployments", "last_auto_restart_at", "TEXT"),
     ]
     for table, column, col_type in migrations:
         if not _has_column(table, column):

app/main.py

@@ -13,6 +13,7 @@ from slowapi.errors import RateLimitExceeded
 from app.database import init_db
 from app.limiter import limiter
 from app.routers import auth, customers, deployments, monitoring, settings, users
+from app.services import scheduler_service
 
 # ---------------------------------------------------------------------------
 # Logging
@@ -33,7 +34,7 @@ logger = logging.getLogger(__name__)
 app = FastAPI(
     title="NetBird MSP Appliance",
     description="Multi-tenant NetBird management platform for MSPs",
-    version="1.2.0",
+    version="1.6.0",
     docs_url="/api/docs",
     redoc_url="/api/redoc",
     openapi_url="/api/openapi.json",
@@ -140,3 +141,10 @@ async def startup_event():
     logger.info("Starting NetBird MSP Appliance...")
     init_db()
     logger.info("Database initialized.")
+    scheduler_service.start()
+
+
+@app.on_event("shutdown")
+async def shutdown_event():
+    """Stop background tasks on shutdown."""
+    scheduler_service.stop()

app/models.py

@@ -1,5 +1,6 @@
 """SQLAlchemy ORM models for NetBird MSP Appliance."""
 
+import json
 from datetime import datetime
 from typing import Optional
 
@@ -88,11 +89,14 @@ class Deployment(Base):
     setup_url: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
     netbird_admin_email: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
     netbird_admin_password: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+    netbird_api_token_encrypted: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+    netbird_api_token_renewed_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
     deployment_status: Mapped[str] = mapped_column(
         String(20), default="pending", nullable=False
     )
     deployed_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
     last_health_check: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
+    last_auto_restart_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
 
     __table_args__ = (
         CheckConstraint(
@@ -116,11 +120,18 @@ class Deployment(Base):
             "relay_secret": "***",  # Never expose secrets
             "setup_url": self.setup_url,
             "has_credentials": bool(self.netbird_admin_email and self.netbird_admin_password),
+            "has_netbird_api_token": bool(self.netbird_api_token_encrypted),
+            "netbird_api_token_renewed_at": (
+                self.netbird_api_token_renewed_at.isoformat() if self.netbird_api_token_renewed_at else None
+            ),
             "deployment_status": self.deployment_status,
             "deployed_at": self.deployed_at.isoformat() if self.deployed_at else None,
             "last_health_check": (
                 self.last_health_check.isoformat() if self.last_health_check else None
             ),
+            "last_auto_restart_at": (
+                self.last_auto_restart_at.isoformat() if self.last_auto_restart_at else None
+            ),
         }
 
 
@@ -199,6 +210,22 @@ class SystemConfig(Base):
     git_branch: Mapped[Optional[str]] = mapped_column(String(100), default="main")
     git_token_encrypted: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
 
+    # Automatic NetBird image update check/apply
+    auto_update_check_enabled: Mapped[bool] = mapped_column(Boolean, default=False)
+    auto_update_check_time: Mapped[Optional[str]] = mapped_column(String(5), default="03:00")
+    auto_update_apply_enabled: Mapped[bool] = mapped_column(Boolean, default=False)
+    auto_update_last_run_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
+
+    # Automatic restart of customer containers when found unreachable/not running
+    auto_restart_unreachable_enabled: Mapped[bool] = mapped_column(Boolean, default=False)
+    auto_restart_cooldown_minutes: Mapped[int] = mapped_column(Integer, default=30)
+
+    # Master default for the NetBird *client* (peer) automatic-updates feature
+    # (Settings > Clients > Automatic Updates inside each customer's own
+    # NetBird dashboard) — pushed to customers via the NetBird Management API.
+    netbird_client_auto_update_version: Mapped[str] = mapped_column(String(50), default="disabled")
+    netbird_client_auto_update_always: Mapped[bool] = mapped_column(Boolean, default=False)
+
     created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
     updated_at: Mapped[datetime] = mapped_column(
         DateTime, default=datetime.utcnow, onupdate=datetime.utcnow
@@ -253,6 +280,16 @@ class SystemConfig(Base):
             "git_repo_url": self.git_repo_url or "",
             "git_branch": self.git_branch or "main",
             "git_token_set": bool(self.git_token_encrypted),
+            "auto_update_check_enabled": bool(self.auto_update_check_enabled),
+            "auto_update_check_time": self.auto_update_check_time or "03:00",
+            "auto_update_apply_enabled": bool(self.auto_update_apply_enabled),
+            "auto_update_last_run_at": (
+                self.auto_update_last_run_at.isoformat() if self.auto_update_last_run_at else None
+            ),
+            "auto_restart_unreachable_enabled": bool(self.auto_restart_unreachable_enabled),
+            "auto_restart_cooldown_minutes": self.auto_restart_cooldown_minutes or 30,
+            "netbird_client_auto_update_version": self.netbird_client_auto_update_version or "disabled",
+            "netbird_client_auto_update_always": bool(self.netbird_client_auto_update_always),
             "created_at": self.created_at.isoformat() if self.created_at else None,
             "updated_at": self.updated_at.isoformat() if self.updated_at else None,
         }
@@ -295,6 +332,56 @@ class DeploymentLog(Base):
         }
 
 
+class UpdateRunLog(Base):
+    """Audit trail for NetBird Docker image update runs (pull + container recreate).
+
+    Written for both scheduled (automatic) and manual (button-triggered)
+    runs, so the actual outcome — which image pulls failed, which customer
+    container recreates failed — is visible in the UI instead of only in
+    container logs.
+    """
+
+    __tablename__ = "update_run_logs"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    run_type: Mapped[str] = mapped_column(String(20), nullable=False)  # "scheduled" | "manual"
+    trigger: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)  # username, or "scheduler"
+    started_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
+    finished_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
+    status: Mapped[str] = mapped_column(
+        String(20), default="running"
+    )  # running | no_update | success | partial | failed
+    any_update_available: Mapped[Optional[bool]] = mapped_column(Boolean, nullable=True)
+    pull_attempted: Mapped[bool] = mapped_column(Boolean, default=False)
+    pull_success: Mapped[Optional[bool]] = mapped_column(Boolean, nullable=True)
+    pull_details: Mapped[Optional[str]] = mapped_column(Text, nullable=True)  # JSON: {image: {success, error}}
+    apply_attempted: Mapped[bool] = mapped_column(Boolean, default=False)
+    customers_updated: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
+    customers_total: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
+    customer_details: Mapped[Optional[str]] = mapped_column(Text, nullable=True)  # JSON list
+    error: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+
+    def to_dict(self) -> dict:
+        """Serialize to dict, parsing the embedded JSON blobs."""
+        return {
+            "id": self.id,
+            "run_type": self.run_type,
+            "trigger": self.trigger,
+            "started_at": self.started_at.isoformat() if self.started_at else None,
+            "finished_at": self.finished_at.isoformat() if self.finished_at else None,
+            "status": self.status,
+            "any_update_available": self.any_update_available,
+            "pull_attempted": self.pull_attempted,
+            "pull_success": self.pull_success,
+            "pull_details": json.loads(self.pull_details) if self.pull_details else None,
+            "apply_attempted": self.apply_attempted,
+            "customers_updated": self.customers_updated,
+            "customers_total": self.customers_total,
+            "customer_details": json.loads(self.customer_details) if self.customer_details else None,
+            "error": self.error,
+        }
+
+
 class User(Base):
     """Admin user model."""
 

app/routers/deployments.py

@@ -1,6 +1,7 @@
 """Deployment management API — start, stop, restart, logs, health for customers."""
 
 import logging
+from datetime import datetime
 
 from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query, status
 from sqlalchemy.orm import Session
@@ -8,8 +9,9 @@ from sqlalchemy.orm import Session
 from app.database import SessionLocal, get_db
 from app.dependencies import get_current_user
 from app.models import Customer, Deployment, SystemConfig, User
-from app.services import docker_service, image_service, netbird_service
+from app.services import docker_service, image_service, netbird_client_update_service, netbird_service, update_log_service
-from app.utils.security import decrypt_value
+from app.utils.security import decrypt_value, encrypt_value
+from app.utils.validators import NetbirdApiTokenPayload, NetbirdClientAutoUpdatePayload
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
@@ -244,9 +246,18 @@ async def update_customer_images(
             status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="System not configured."
         )
 
+    started_at = datetime.utcnow()
     instance_dir = f"{config.data_dir}/{customer.subdomain}"
     result = await image_service.update_customer_containers(instance_dir, deployment.container_prefix)
 
+    update_log_service.record_run(
+        db, run_type="manual", trigger=current_user.username, started_at=started_at,
+        apply_attempted=True,
+        customer_results=[{
+            "customer_name": customer.name, "success": result["success"], "error": result.get("error"),
+        }],
+    )
+
     if not result["success"]:
         raise HTTPException(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
@@ -260,6 +271,117 @@ async def update_customer_images(
     return {"message": f"Containers updated for '{customer.name}'."}
 
 
+@router.get("/{customer_id}/netbird-updates")
+async def get_customer_netbird_updates(
+    customer_id: int,
+    current_user: User = Depends(get_current_user),
+    db: Session = Depends(get_db),
+):
+    """Fetch a customer's *live* NetBird client automatic-updates setting.
+
+    Reads directly from the customer's NetBird Management API — always
+    reflects reality, including changes made manually in their own dashboard.
+    """
+    _require_customer(db, customer_id)
+    deployment = db.query(Deployment).filter(Deployment.customer_id == customer_id).first()
+    if not deployment:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No deployment found for this customer.")
+    if not deployment.netbird_api_token_encrypted:
+        return {"has_token": False, "version": None, "always": None}
+
+    token = decrypt_value(deployment.netbird_api_token_encrypted)
+    result = await netbird_client_update_service.get_current_settings(deployment.container_prefix, token)
+    if not result["ok"]:
+        raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=result["error"])
+
+    settings = result["settings"]
+    return {
+        "has_token": True,
+        "token_renewed_at": (
+            deployment.netbird_api_token_renewed_at.isoformat()
+            if deployment.netbird_api_token_renewed_at else None
+        ),
+        "version": settings.get("auto_update_version", "disabled"),
+        "always": bool(settings.get("auto_update_always", False)),
+    }
+
+
+@router.put("/{customer_id}/netbird-updates")
+async def set_customer_netbird_updates(
+    customer_id: int,
+    payload: NetbirdClientAutoUpdatePayload,
+    current_user: User = Depends(get_current_user),
+    db: Session = Depends(get_db),
+):
+    """Push a client automatic-updates version/mode to a single customer.
+
+    Use this to override the master default for one customer specifically —
+    e.g. a customer on a legacy client that must not jump straight to latest.
+    """
+    if current_user.role != "admin":
+        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin only.")
+
+    _require_customer(db, customer_id)
+    deployment = db.query(Deployment).filter(Deployment.customer_id == customer_id).first()
+    if not deployment:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No deployment found for this customer.")
+    if not deployment.netbird_api_token_encrypted:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="No NetBird API token registered for this customer. Paste one via PUT .../netbird-api-token first.",
+        )
+
+    token = decrypt_value(deployment.netbird_api_token_encrypted)
+    result = await netbird_client_update_service.push_auto_update_settings(
+        deployment.container_prefix, token, payload.version, payload.always
+    )
+    if not result["ok"]:
+        raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=result["error"])
+
+    logger.info(
+        "NetBird client auto-update set for customer %d (%s): version=%s always=%s by %s",
+        customer_id, deployment.container_prefix, payload.version, payload.always, current_user.username,
+    )
+    return {"ok": True}
+
+
+@router.put("/{customer_id}/netbird-api-token")
+async def set_customer_netbird_api_token(
+    customer_id: int,
+    payload: NetbirdApiTokenPayload,
+    current_user: User = Depends(get_current_user),
+    db: Session = Depends(get_db),
+):
+    """Manually register a NetBird Personal Access Token for a customer.
+
+    Needed for customers deployed before automatic PAT capture — create a
+    PAT once in that customer's dashboard (Settings > Service Users /
+    Personal Access Tokens) and paste it here. New deployments capture one
+    automatically during setup.
+    """
+    if current_user.role != "admin":
+        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin only.")
+
+    _require_customer(db, customer_id)
+    deployment = db.query(Deployment).filter(Deployment.customer_id == customer_id).first()
+    if not deployment:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No deployment found for this customer.")
+
+    # Validate the token actually works before storing it.
+    result = await netbird_client_update_service.get_current_settings(deployment.container_prefix, payload.token)
+    if not result["ok"]:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"Token could not be verified against this customer's NetBird instance: {result['error']}",
+        )
+
+    deployment.netbird_api_token_encrypted = encrypt_value(payload.token)
+    deployment.netbird_api_token_renewed_at = datetime.utcnow()
+    db.commit()
+    logger.info("NetBird API token registered for customer %d by %s.", customer_id, current_user.username)
+    return {"ok": True}
+
+
 def _require_customer(db: Session, customer_id: int) -> Customer:
     """Helper to fetch a customer or raise 404.
 

app/routers/monitoring.py

@@ -4,6 +4,7 @@ import asyncio
 import logging
 import platform
 import time
+from datetime import datetime
 from typing import Any
 
 import psutil
@@ -12,8 +13,10 @@ from sqlalchemy.orm import Session
 
 from app.database import SessionLocal, get_db
 from app.dependencies import get_current_user
-from app.models import Customer, Deployment, SystemConfig, User
+from app.models import Customer, Deployment, SystemConfig, UpdateRunLog, User
-from app.services import docker_service, image_service
+from app.services import docker_service, image_service, netbird_client_update_service, update_log_service
+from app.utils.security import decrypt_value
+from app.utils.validators import NetbirdClientAutoUpdatePayload
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
@@ -159,6 +162,7 @@ async def check_image_updates(
             "subdomain": customer.subdomain,
             "container_prefix": dep.container_prefix,
             "needs_update": cs["needs_update"],
+            "unknown": cs.get("unknown", False),
             "services": cs["services"],
         })
 
@@ -191,14 +195,25 @@ async def pull_all_netbird_images(
         config.netbird_dashboard_image,
     ]
 
+    username = current_user.username
+
     async def _pull_bg() -> None:
+        started_at = datetime.utcnow()
         bg_db = SessionLocal()
         try:
             cfg = bg_db.query(SystemConfig).filter(SystemConfig.id == 1).first()
-            if cfg:
+            if not cfg:
-                await image_service.pull_all_images(cfg)
+                return
-        except Exception:
+            pull_result = await image_service.pull_all_images(cfg)
+            update_log_service.record_run(
+                bg_db, run_type="manual", trigger=username, started_at=started_at,
+                any_update_available=True, pull_attempted=True, pull_results=pull_result["results"],
+            )
+        except Exception as exc:
             logger.exception("Background image pull failed")
+            update_log_service.record_run(
+                bg_db, run_type="manual", trigger=username, started_at=started_at, error=str(exc),
+            )
         finally:
             bg_db.close()
 
@@ -231,7 +246,7 @@ async def customers_local_update_status(
 
     async def _check(dep: Deployment) -> dict[str, Any]:
         cs = await image_service.get_customer_container_image_status_async(dep.container_prefix, config)
-        return {"customer_id": dep.customer_id, "needs_update": cs["needs_update"]}
+        return {"customer_id": dep.customer_id, "needs_update": cs["needs_update"], "unknown": cs.get("unknown", False)}
 
     results = await asyncio.gather(*[_check(dep) for dep in deployments])
     results = list(results)
@@ -240,6 +255,70 @@ async def customers_local_update_status(
     return results
 
 
+@router.post("/netbird-updates/apply-all")
+async def apply_netbird_client_updates_to_all(
+    payload: NetbirdClientAutoUpdatePayload,
+    current_user: User = Depends(get_current_user),
+    db: Session = Depends(get_db),
+) -> dict[str, Any]:
+    """Push a NetBird client automatic-updates version/mode to every customer.
+
+    Skips (and reports) customers without a registered API token — they need
+    a token pasted in via the per-customer endpoint first (older deployments
+    predating automatic token capture).
+    """
+    if current_user.role != "admin":
+        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin only.")
+
+    deployments = db.query(Deployment).all()
+    results = []
+    for dep in deployments:
+        customer = dep.customer
+        if not dep.netbird_api_token_encrypted:
+            results.append({
+                "customer_id": customer.id, "customer_name": customer.name,
+                "success": False, "error": "No API token registered.",
+            })
+            continue
+        token = decrypt_value(dep.netbird_api_token_encrypted)
+        res = await netbird_client_update_service.push_auto_update_settings(
+            dep.container_prefix, token, payload.version, payload.always
+        )
+        results.append({
+            "customer_id": customer.id, "customer_name": customer.name,
+            "success": res["ok"], "error": res.get("error"),
+        })
+
+    success_count = sum(1 for r in results if r["success"])
+    return {
+        "message": f"Applied to {success_count} of {len(results)} customer(s).",
+        "updated": success_count,
+        "results": results,
+    }
+
+
+@router.get("/update-logs")
+async def get_update_logs(
+    limit: int = 30,
+    current_user: User = Depends(get_current_user),
+    db: Session = Depends(get_db),
+) -> list[dict[str, Any]]:
+    """Return the most recent NetBird image update runs (scheduled and manual).
+
+    Shows exactly what happened on each run: whether an image pull was
+    attempted and its per-image result, and whether customer containers
+    were recreated and which ones failed — so this doesn't have to be
+    reconstructed from container logs.
+    """
+    logs = (
+        db.query(UpdateRunLog)
+        .order_by(UpdateRunLog.started_at.desc())
+        .limit(min(limit, 100))
+        .all()
+    )
+    return [log.to_dict() for log in logs]
+
+
 @router.post("/customers/update-all")
 async def update_all_customers(
     current_user: User = Depends(get_current_user),
@@ -254,6 +333,7 @@ async def update_all_customers(
     if current_user.role != "admin":
         raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin only.")
 
+    started_at = datetime.utcnow()
     config = db.query(SystemConfig).filter(SystemConfig.id == 1).first()
     if not config:
         raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="System not configured.")
@@ -272,24 +352,40 @@ async def update_all_customers(
             })
 
     if not to_update:
+        update_log_service.record_run(
+            db, run_type="manual", trigger=current_user.username,
+            started_at=started_at, apply_attempted=True, customer_results=[],
+        )
         return {"message": "All customers are already up to date.", "updated": 0, "results": []}
 
-    # Update customers sequentially — one at a time
+    # Update customers sequentially — one at a time. A failure for one
+    # customer (e.g. a hung docker compose call) must not abort the rest of
+    # the batch, otherwise later customers silently never get updated.
     update_results = []
     for entry in to_update:
+        try:
             res = await image_service.update_customer_containers(
                 entry["instance_dir"], entry["project_name"]
             )
             ok = res["success"]
-        logger.info("Updated %s: %s", entry["project_name"], "OK" if ok else res.get("error"))
+            error = res.get("error")
+        except Exception as exc:
+            logger.exception("Unexpected error updating %s", entry["project_name"])
+            ok = False
+            error = str(exc)
+        logger.info("Updated %s: %s", entry["project_name"], "OK" if ok else error)
         update_results.append({
             "customer_name": entry["customer_name"],
             "customer_id": entry["customer_id"],
             "success": ok,
-            "error": res.get("error"),
+            "error": error,
         })
 
     success_count = sum(1 for r in update_results if r["success"])
+    update_log_service.record_run(
+        db, run_type="manual", trigger=current_user.username,
+        started_at=started_at, apply_attempted=True, customer_results=update_results,
+    )
     return {
         "message": f"Updated {success_count} of {len(update_results)} customer(s).",
         "updated": success_count,

app/services/image_service.py

@@ -19,13 +19,38 @@ logger = logging.getLogger(__name__)
 NETBIRD_SERVICES = ["management", "signal", "relay", "dashboard"]
 
 
+class _TimeoutResult:
+    """Stand-in for subprocess.CompletedProcess when a command times out.
+
+    A hung `docker compose up -d` used to raise TimeoutExpired straight out of
+    _run_cmd, which killed the whole update-all loop mid-recreate and left the
+    old container renamed-but-not-removed (orphaned with a hash-prefixed name).
+    Returning a failed result instead lets callers handle it gracefully and
+    keeps the batch loop going for the remaining customers.
+    """
+
+    def __init__(self, cmd: list[str], timeout: int):
+        self.returncode = -1
+        self.stdout = ""
+        self.stderr = f"Command timed out after {timeout}s: {' '.join(cmd)}"
+
+
 async def _run_cmd(cmd: list[str], timeout: int = 300) -> subprocess.CompletedProcess:
-    """Run a subprocess command without blocking the event loop."""
+    """Run a subprocess command without blocking the event loop.
+
+    Never raises on timeout — returns a failed CompletedProcess-like result
+    instead, so a single hung docker/compose call can't abort a batch of
+    otherwise-independent operations (e.g. updating multiple customers).
+    """
     loop = asyncio.get_event_loop()
+    try:
         return await loop.run_in_executor(
             None,
             lambda: subprocess.run(cmd, capture_output=True, text=True, timeout=timeout),
         )
+    except subprocess.TimeoutExpired:
+        logger.error("Command timed out after %ds: %s", timeout, " ".join(cmd))
+        return _TimeoutResult(cmd, timeout)
 
 
 def _parse_image_name(image: str) -> tuple[str, str]:
@@ -123,6 +148,60 @@ def get_container_image_id(container_name: str) -> str | None:
         return None
 
 
+def repair_container_naming(container_prefix: str, services: list[str] = NETBIRD_SERVICES) -> list[str]:
+    """Rename orphaned containers back to their expected compose name.
+
+    When a `docker compose up -d` is interrupted mid-recreate (e.g. a timeout
+    killing the process), Compose can leave the *old* container renamed with a
+    random hash prefix (e.g. "4e45e71fcb7b_netbird-acme-management") instead
+    of removing it, while never creating the correctly-named replacement. The
+    container itself keeps running fine — it's just invisible to every lookup
+    that expects the exact name, which used to silently read as "no container
+    found" and get reported as "up to date" instead of "unknown".
+
+    This finds any such orphan (a container whose name *contains* the expected
+    name but isn't an exact match) and, only when no container already holds
+    the exact expected name, renames it back. Safe no-op otherwise.
+
+    Returns the list of service names that were repaired.
+    """
+    repaired = []
+    for svc in services:
+        expected_name = f"{container_prefix}-{svc}"
+        exact = subprocess.run(
+            ["docker", "inspect", expected_name, "--format", "{{.Id}}"],
+            capture_output=True, text=True, timeout=10,
+        )
+        if exact.returncode == 0:
+            continue  # already correctly named
+
+        found = subprocess.run(
+            ["docker", "ps", "-a", "--filter", f"name={expected_name}", "--format", "{{.Names}}"],
+            capture_output=True, text=True, timeout=10,
+        )
+        candidates = [n for n in found.stdout.strip().splitlines() if n and n != expected_name]
+        if not candidates:
+            continue  # container genuinely doesn't exist (not deployed / not running)
+
+        orphan = candidates[0]
+        rename = subprocess.run(
+            ["docker", "rename", orphan, expected_name],
+            capture_output=True, text=True, timeout=10,
+        )
+        if rename.returncode == 0:
+            logger.warning(
+                "Repaired orphaned container naming for %s: '%s' -> '%s'",
+                container_prefix, orphan, expected_name,
+            )
+            repaired.append(svc)
+        else:
+            logger.error(
+                "Failed to repair orphaned container '%s' -> '%s': %s",
+                orphan, expected_name, rename.stderr,
+            )
+    return repaired
+
+
 def get_local_image_id(image: str) -> str | None:
     """Get the full image ID (sha256:...) of a locally stored image."""
     try:
@@ -231,6 +310,13 @@ async def get_customer_container_image_status_async(container_prefix: str, confi
     }
     loop = asyncio.get_event_loop()
 
+    # Self-heal any container left orphaned under a hash-prefixed name by a
+    # previously interrupted recreate, so the lookups below find it by its
+    # real, expected name instead of silently returning "not found".
+    await loop.run_in_executor(
+        None, repair_container_naming, container_prefix, list(service_images.keys())
+    )
+
     async def _check(svc: str, image: str) -> tuple[str, dict[str, Any]]:
         container_name = f"{container_prefix}-{svc}"
         container_id, local_id = await asyncio.gather(
@@ -246,7 +332,8 @@ async def get_customer_container_image_status_async(container_prefix: str, confi
     pairs = await asyncio.gather(*[_check(svc, image) for svc, image in service_images.items()])
     services = dict(pairs)
     needs_update = any(s["up_to_date"] is False for s in services.values())
-    return {"services": services, "needs_update": needs_update}
+    unknown = any(s["up_to_date"] is None for s in services.values())
+    return {"services": services, "needs_update": needs_update, "unknown": unknown}
 
 
 def get_customer_container_image_status(container_prefix: str, config) -> dict[str, Any]:
@@ -265,6 +352,11 @@ def get_customer_container_image_status(container_prefix: str, config) -> dict[s
         "relay": config.netbird_relay_image,
         "dashboard": config.netbird_dashboard_image,
     }
+
+    # Self-heal any container left orphaned under a hash-prefixed name by a
+    # previously interrupted recreate (see repair_container_naming docstring).
+    repair_container_naming(container_prefix, list(service_images.keys()))
+
     services: dict[str, Any] = {}
     for svc, image in service_images.items():
         container_name = f"{container_prefix}-{svc}"
@@ -280,7 +372,8 @@ def get_customer_container_image_status(container_prefix: str, config) -> dict[s
             "up_to_date": up_to_date,
         }
     needs_update = any(s["up_to_date"] is False for s in services.values())
-    return {"services": services, "needs_update": needs_update}
+    unknown = any(s["up_to_date"] is None for s in services.values())
+    return {"services": services, "needs_update": needs_update, "unknown": unknown}
 
 
 async def update_customer_containers(instance_dir: str, project_name: str) -> dict[str, Any]:
@@ -292,6 +385,13 @@ async def update_customer_containers(instance_dir: str, project_name: str) -> di
     compose_file = os.path.join(instance_dir, "docker-compose.yml")
     if not os.path.isfile(compose_file):
         return {"success": False, "error": f"docker-compose.yml not found at {compose_file}"}
+
+    # Repair any container still orphaned under a hash-prefixed name from a
+    # previous interrupted recreate before Compose tries to touch it again —
+    # otherwise Compose keeps colliding with the same stuck rename.
+    loop = asyncio.get_event_loop()
+    await loop.run_in_executor(None, repair_container_naming, project_name)
+
     cmd = [
         "docker", "compose",
         "-f", compose_file,

app/services/netbird_client_update_service.py

@@ -0,0 +1,129 @@
+"""Central control of the NetBird *client* (peer) Automatic Updates feature.
+
+This is the "Settings > Clients > Automatic Updates" toggle inside each
+customer's own NetBird dashboard (netbirdio/netbird, added in v0.61.0) — not
+to be confused with updating the NetBird Docker images themselves
+(app/services/image_service.py).
+
+Talked to over the customer's NetBird Management REST API, authenticated
+with a Personal Access Token captured during initial deployment (see
+netbird_service.deploy_customer) or pasted in manually for customers
+deployed before this feature existed. Requests go over the internal Docker
+network directly to the customer's management container — never through
+their public dashboard URL.
+"""
+
+import json
+import logging
+from typing import Any
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+_TIMEOUT = 10
+
+
+def _base_url(container_prefix: str) -> str:
+    return f"http://{container_prefix}-management:80"
+
+
+async def get_current_settings(container_prefix: str, token: str) -> dict[str, Any]:
+    """Fetch the customer's current account settings.
+
+    Returns:
+        {"ok": True, "account_id": ..., "settings": {...}} on success, or
+        {"ok": False, "error": "..."} on failure.
+    """
+    base_url = _base_url(container_prefix)
+    headers = {"Authorization": f"Token {token}"}
+    try:
+        async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
+            resp = await client.get(f"{base_url}/api/accounts", headers=headers)
+            if resp.status_code != 200:
+                return {"ok": False, "error": f"GET /api/accounts -> HTTP {resp.status_code}: {resp.text[:300]}"}
+            accounts = resp.json()
+            if not accounts:
+                return {"ok": False, "error": "No account returned by /api/accounts."}
+            account = accounts[0]
+            return {"ok": True, "account_id": account["id"], "settings": account.get("settings", {})}
+    except Exception as exc:
+        logger.warning("Failed to fetch NetBird account settings for %s: %s", container_prefix, exc)
+        return {"ok": False, "error": str(exc)}
+
+
+async def renew_token(container_prefix: str, token: str) -> dict[str, Any]:
+    """Mint a fresh 365-day Personal Access Token using the current one.
+
+    The old token is left in place (it naturally expires and NetBird gives no
+    reliable way to identify "our" token among a user's other PATs by content
+    alone, only by name — which isn't safe to assume is unique). One
+    harmless unused token lingering until its own expiry is an acceptable
+    trade-off for not risking deleting a token that turns out to be in use.
+    """
+    base_url = _base_url(container_prefix)
+    headers = {"Authorization": f"Token {token}"}
+    try:
+        async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
+            resp = await client.get(f"{base_url}/api/users", headers=headers)
+            if resp.status_code != 200:
+                return {"ok": False, "error": f"GET /api/users -> HTTP {resp.status_code}"}
+            users = resp.json()
+            me = next((u for u in users if u.get("is_current")), None)
+            if not me:
+                return {"ok": False, "error": "Could not identify current user from /api/users."}
+
+            create_resp = await client.post(
+                f"{base_url}/api/users/{me['id']}/tokens",
+                headers={**headers, "Content-Type": "application/json"},
+                content=json.dumps({"name": "MSP Central Management", "expires_in": 365}),
+            )
+            if create_resp.status_code not in (200, 201):
+                return {"ok": False, "error": f"POST tokens -> HTTP {create_resp.status_code}: {create_resp.text[:200]}"}
+            new_token = create_resp.json().get("plain_token")
+            if not new_token:
+                return {"ok": False, "error": "Token creation response had no plain_token."}
+
+            verify_resp = await client.get(f"{base_url}/api/accounts", headers={"Authorization": f"Token {new_token}"})
+            if verify_resp.status_code != 200:
+                return {"ok": False, "error": "New token failed verification."}
+
+            return {"ok": True, "token": new_token}
+    except Exception as exc:
+        logger.warning("Failed to renew NetBird API token for %s: %s", container_prefix, exc)
+        return {"ok": False, "error": str(exc)}
+
+
+async def push_auto_update_settings(
+    container_prefix: str, token: str, version: str, always: bool
+) -> dict[str, Any]:
+    """Set the client automatic-updates version/mode for one customer.
+
+    NetBird's account PUT endpoint expects the *entire* settings object, not
+    a partial patch, so this fetches current settings first and only
+    overwrites the two auto-update fields.
+    """
+    current = await get_current_settings(container_prefix, token)
+    if not current["ok"]:
+        return current
+
+    settings = dict(current["settings"])
+    settings["auto_update_version"] = version
+    settings["auto_update_always"] = always
+
+    base_url = _base_url(container_prefix)
+    account_id = current["account_id"]
+    headers = {"Authorization": f"Token {token}", "Content-Type": "application/json"}
+    body = {"settings": settings}
+
+    try:
+        async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
+            resp = await client.put(
+                f"{base_url}/api/accounts/{account_id}", headers=headers, content=json.dumps(body)
+            )
+            if resp.status_code != 200:
+                return {"ok": False, "error": f"PUT /api/accounts/{account_id} -> HTTP {resp.status_code}: {resp.text[:300]}"}
+            return {"ok": True}
+    except Exception as exc:
+        logger.warning("Failed to push NetBird auto-update settings for %s: %s", container_prefix, exc)
+        return {"ok": False, "error": str(exc)}

app/services/netbird_service.py

@@ -26,6 +26,7 @@ import urllib.error
 from datetime import datetime
 from typing import Any
 
+import httpx
 from jinja2 import Environment, FileSystemLoader
 from sqlalchemy.orm import Session
 
@@ -231,9 +232,12 @@ async def deploy_customer(db: Session, customer_id: int) -> dict[str, Any]:
             "name": customer.name,
             "email": admin_email,
             "password": admin_password,
+            "create_pat": True,
+            "pat_expire_in": 365,
         }).encode("utf-8")
 
         setup_ok = False
+        netbird_api_token: str | None = None
         for attempt in range(10):
             try:
                 req = urllib.request.Request(
@@ -245,8 +249,13 @@ async def deploy_customer(db: Session, customer_id: int) -> dict[str, Any]:
                 with urllib.request.urlopen(req, timeout=10) as resp:
                     if resp.status in (200, 201):
                         setup_ok = True
+                        setup_body = json.loads(resp.read().decode("utf-8"))
+                        netbird_api_token = setup_body.get("personal_access_token")
                         _log_action(db, customer_id, "deploy", "info",
-                                    f"Admin user created: {admin_email}")
+                                    f"Admin user created: {admin_email}"
+                                    + (" (API token captured for central management)"
+                                       if netbird_api_token else
+                                       " (no API token — NB_SETUP_PAT_ENABLED not active yet on this instance)"))
                         break
             except urllib.error.HTTPError as e:
                 body = e.read().decode("utf-8", errors="replace")
@@ -340,6 +349,9 @@ async def deploy_customer(db: Session, customer_id: int) -> dict[str, Any]:
             deployment.setup_url = setup_url
             deployment.netbird_admin_email = encrypt_value(admin_email) if setup_ok else deployment.netbird_admin_email
             deployment.netbird_admin_password = encrypt_value(admin_password) if setup_ok else deployment.netbird_admin_password
+            if netbird_api_token:
+                deployment.netbird_api_token_encrypted = encrypt_value(netbird_api_token)
+                deployment.netbird_api_token_renewed_at = datetime.utcnow()
             deployment.deployment_status = "running"
             deployment.deployed_at = datetime.utcnow()
         else:
@@ -354,6 +366,8 @@ async def deploy_customer(db: Session, customer_id: int) -> dict[str, Any]:
                 setup_url=setup_url,
                 netbird_admin_email=encrypt_value(admin_email) if setup_ok else None,
                 netbird_admin_password=encrypt_value(admin_password) if setup_ok else None,
+                netbird_api_token_encrypted=encrypt_value(netbird_api_token) if netbird_api_token else None,
+                netbird_api_token_renewed_at=datetime.utcnow() if netbird_api_token else None,
                 deployment_status="running",
                 deployed_at=datetime.utcnow(),
             )
@@ -516,6 +530,25 @@ async def restart_customer(db: Session, customer_id: int) -> dict[str, Any]:
     return {"success": ok}
 
 
+async def check_dashboard_reachable(setup_url: str, timeout: float = 10) -> bool:
+    """Check whether a customer's NetBird dashboard actually responds over HTTP(S).
+
+    A container can be "running" in Docker while the web UI behind it is
+    still unreachable (e.g. the management service is up but not accepting
+    connections yet), so this checks the real endpoint rather than the
+    container state. Any HTTP response (including 4xx/5xx from the app
+    itself) counts as reachable — only connection failures/timeouts don't.
+    """
+    if not setup_url:
+        return True  # nothing to check (e.g. deployment not fully set up yet)
+    try:
+        async with httpx.AsyncClient(timeout=timeout) as client:
+            await client.get(setup_url)
+        return True
+    except httpx.HTTPError:
+        return False
+
+
 def get_customer_health(db: Session, customer_id: int) -> dict[str, Any]:
     """Check health of a customer's deployment."""
     deployment = db.query(Deployment).filter(Deployment.customer_id == customer_id).first()

app/services/scheduler_service.py

@@ -0,0 +1,283 @@
+"""Background scheduler for automatic NetBird image update checks.
+
+No external scheduler dependency (APScheduler etc.) — a single asyncio task
+started at app startup wakes up once a minute, and only actually does
+anything once per day at the configured HH:MM, controlled entirely by
+SystemConfig.auto_update_check_enabled / auto_update_check_time.
+"""
+
+import asyncio
+import logging
+from datetime import datetime, timedelta
+
+from app.database import SessionLocal
+from app.models import Deployment, SystemConfig
+from app.services import (
+    docker_service,
+    image_service,
+    netbird_client_update_service,
+    netbird_service,
+    update_log_service,
+)
+from app.services.netbird_service import _log_action
+from app.utils.security import decrypt_value, encrypt_value
+
+logger = logging.getLogger(__name__)
+
+_POLL_INTERVAL_SECONDS = 60
+_task: asyncio.Task | None = None
+
+# NetBird PATs we mint are issued for 365 days; renew well before that so a
+# missed tick or a slow rollout never risks the token actually expiring.
+_TOKEN_RENEW_AFTER_DAYS = 300
+_TOKEN_RENEW_CHECK_HOUR = 4  # run once per day, distinct from the image-check hour
+
+
+def start() -> None:
+    """Start the background polling task. Safe to call once at app startup."""
+    global _task
+    if _task is None or _task.done():
+        _task = asyncio.create_task(_poll_loop())
+        logger.info("Automatic update scheduler started.")
+
+
+def stop() -> None:
+    """Cancel the background polling task."""
+    global _task
+    if _task is not None:
+        _task.cancel()
+        _task = None
+
+
+_last_token_renewal_date = None
+
+
+async def _poll_loop() -> None:
+    while True:
+        try:
+            await _tick()
+        except Exception:
+            logger.exception("Scheduler tick failed")
+        try:
+            await _token_renewal_tick()
+        except Exception:
+            logger.exception("Token renewal tick failed")
+        try:
+            await _auto_restart_tick()
+        except Exception:
+            logger.exception("Auto-restart tick failed")
+        await asyncio.sleep(_POLL_INTERVAL_SECONDS)
+
+
+async def _token_renewal_tick() -> None:
+    """Once a day, renew any NetBird client-update API token nearing its
+    365-day expiry — keeps central update control working indefinitely
+    without anyone needing to notice or act.
+    """
+    global _last_token_renewal_date
+    now = datetime.now()
+    if now.hour != _TOKEN_RENEW_CHECK_HOUR:
+        return
+    if _last_token_renewal_date == now.date():
+        return
+    _last_token_renewal_date = now.date()
+
+    db = SessionLocal()
+    try:
+        cutoff = now - timedelta(days=_TOKEN_RENEW_AFTER_DAYS)
+        deployments = (
+            db.query(Deployment)
+            .filter(Deployment.netbird_api_token_encrypted.isnot(None))
+            .all()
+        )
+        due = [
+            d for d in deployments
+            if d.netbird_api_token_renewed_at is None or d.netbird_api_token_renewed_at < cutoff
+        ]
+        if not due:
+            return
+        logger.info("Renewing NetBird API token for %d customer(s)...", len(due))
+        for d in due:
+            token = decrypt_value(d.netbird_api_token_encrypted)
+            result = await netbird_client_update_service.renew_token(d.container_prefix, token)
+            if result["ok"]:
+                d.netbird_api_token_encrypted = encrypt_value(result["token"])
+                d.netbird_api_token_renewed_at = now
+                db.commit()
+                logger.info("Renewed NetBird API token for %s.", d.container_prefix)
+            else:
+                logger.warning("Failed to renew NetBird API token for %s: %s", d.container_prefix, result.get("error"))
+    finally:
+        db.close()
+
+
+async def _tick() -> None:
+    db = SessionLocal()
+    try:
+        config = db.query(SystemConfig).filter(SystemConfig.id == 1).first()
+        if not config or not config.auto_update_check_enabled:
+            return
+
+        now = datetime.now()
+        target_time = config.auto_update_check_time or "03:00"
+        current_hhmm = now.strftime("%H:%M")
+        if current_hhmm != target_time:
+            return
+
+        last_run = config.auto_update_last_run_at
+        if last_run and last_run.date() == now.date():
+            return  # already ran today
+
+        # Claim this run immediately so a slow run can't overlap the next tick.
+        config.auto_update_last_run_at = now
+        db.commit()
+
+        apply_enabled = bool(config.auto_update_apply_enabled)
+        logger.info(
+            "Running scheduled NetBird image update check (auto-apply=%s)...", apply_enabled
+        )
+        await _run_check_and_optionally_apply(config, apply_enabled, started_at=now)
+    finally:
+        db.close()
+
+
+async def _run_check_and_optionally_apply(config: SystemConfig, apply_enabled: bool, started_at: datetime) -> None:
+    """Check Docker Hub, pull if needed, then (if enabled) recreate any
+    customer container that isn't running the locally-pulled image yet.
+
+    The "recreate" step is intentionally NOT gated on a Hub update having
+    just been found: a customer can already be behind the *locally pulled*
+    image from an earlier check/pull that was never applied (e.g. auto-apply
+    was off at the time, or a previous run failed) — that backlog must keep
+    being cleared on every run, not just on runs that also found something
+    new on Hub, otherwise it silently never happens.
+    """
+    log_db = SessionLocal()
+    try:
+        hub_status = await image_service.check_all_images(config)
+        any_update = hub_status["any_update_available"]
+
+        pull_attempted = False
+        pull_results = None
+        if any_update:
+            logger.info("Scheduled check: new NetBird image(s) available — pulling.")
+            pull_attempted = True
+            pull_result = await image_service.pull_all_images(config)
+            pull_results = pull_result["results"]
+            if not pull_result["all_success"]:
+                logger.error("Scheduled image pull had failures: %s", pull_results)
+        else:
+            logger.info("Scheduled check: all NetBird images already up to date on Docker Hub.")
+
+        if not apply_enabled:
+            logger.info("Auto-apply disabled — customer containers left untouched.")
+            update_log_service.record_run(
+                log_db, run_type="scheduled", trigger="scheduler",
+                started_at=started_at, any_update_available=any_update,
+                pull_attempted=pull_attempted, pull_results=pull_results,
+            )
+            return
+
+        deployments = log_db.query(Deployment).all()
+        to_update = []
+        for dep in deployments:
+            cs = image_service.get_customer_container_image_status(dep.container_prefix, config)
+            if cs["needs_update"]:
+                customer = dep.customer
+                to_update.append({
+                    "instance_dir": f"{config.data_dir}/{customer.subdomain}",
+                    "project_name": dep.container_prefix,
+                    "customer_name": customer.name,
+                })
+        logger.info("Scheduled auto-apply: updating %d customer(s)...", len(to_update))
+        customer_results = []
+        for entry in to_update:
+            try:
+                res = await image_service.update_customer_containers(
+                    entry["instance_dir"], entry["project_name"]
+                )
+                ok = res["success"]
+                err = res.get("error")
+            except Exception as exc:
+                logger.exception("Scheduled update failed for %s", entry["customer_name"])
+                ok = False
+                err = str(exc)
+            logger.info("Scheduled update for %s: %s", entry["customer_name"], "OK" if ok else err)
+            customer_results.append({"customer_name": entry["customer_name"], "success": ok, "error": err})
+
+        update_log_service.record_run(
+            log_db, run_type="scheduled", trigger="scheduler",
+            started_at=started_at, any_update_available=any_update,
+            pull_attempted=pull_attempted, pull_results=pull_results,
+            apply_attempted=True, customer_results=customer_results,
+        )
+    except Exception as exc:
+        logger.exception("Scheduled update run failed")
+        update_log_service.record_run(
+            log_db, run_type="scheduled", trigger="scheduler",
+            started_at=started_at, error=str(exc),
+        )
+    finally:
+        log_db.close()
+
+
+async def _auto_restart_tick() -> None:
+    """Every poll interval, restart any customer whose NetBird dashboard
+    doesn't actually respond over HTTP(S) — a container can be "running" in
+    Docker while the web UI behind it is still unreachable, which is the
+    actual symptom customers hit, so this probes the real endpoint rather
+    than trusting container state alone. Falls back to container status
+    only if a customer has no setup_url yet. Guarded by
+    SystemConfig.auto_restart_unreachable_enabled and a per-deployment
+    cooldown so a persistently broken customer doesn't get restarted in a
+    tight loop, and skipped for deployments that aren't running/failed
+    (e.g. intentionally stopped or still pending).
+    """
+    db = SessionLocal()
+    try:
+        config = db.query(SystemConfig).filter(SystemConfig.id == 1).first()
+        if not config or not config.auto_restart_unreachable_enabled:
+            return
+
+        cooldown = timedelta(minutes=config.auto_restart_cooldown_minutes or 30)
+        now = datetime.now()
+
+        deployments = db.query(Deployment).all()
+        for dep in deployments:
+            if dep.deployment_status not in ("running", "failed"):
+                continue
+            if dep.last_auto_restart_at and now - dep.last_auto_restart_at < cooldown:
+                continue
+
+            reason = None
+            if dep.setup_url:
+                if not await netbird_service.check_dashboard_reachable(dep.setup_url):
+                    reason = f"dashboard ({dep.setup_url}) did not respond"
+            else:
+                containers = await docker_service.get_container_status_async(dep.container_prefix)
+                if not containers or not all(c["status"] == "running" for c in containers):
+                    not_running = [c["name"] for c in containers if c["status"] != "running"] or ["no containers found"]
+                    reason = f"container(s) not running: {', '.join(not_running)}"
+
+            if not reason:
+                continue
+
+            logger.warning(
+                "Auto-restart: customer %s unreachable (%s) — restarting.",
+                dep.container_prefix, reason,
+            )
+            _log_action(
+                db, dep.customer_id, "auto_restart", "info",
+                f"Automatic restart triggered: {reason}.",
+            )
+            try:
+                result = await netbird_service.restart_customer(db, dep.customer_id)
+                if not result["success"]:
+                    logger.error("Auto-restart failed for customer %s.", dep.container_prefix)
+            except Exception:
+                logger.exception("Auto-restart failed for customer %s.", dep.container_prefix)
+
+            dep.last_auto_restart_at = now
+            db.commit()
+    finally:
+        db.close()

app/services/update_log_service.py

@@ -0,0 +1,84 @@
+"""Persists the outcome of every NetBird Docker image update run.
+
+Both the automatic scheduler and the manual "Check/Pull/Update" buttons in
+the UI write here, so it's visible afterwards which run pulled which image
+successfully, and which customer container recreate failed — instead of
+that information only existing in container logs.
+"""
+
+import json
+import logging
+from datetime import datetime
+from typing import Any, Optional
+
+from sqlalchemy.orm import Session
+
+from app.models import UpdateRunLog
+
+logger = logging.getLogger(__name__)
+
+
+def record_run(
+    db: Session,
+    *,
+    run_type: str,
+    trigger: Optional[str],
+    started_at: datetime,
+    any_update_available: Optional[bool] = None,
+    pull_attempted: bool = False,
+    pull_results: Optional[dict[str, Any]] = None,
+    apply_attempted: bool = False,
+    customer_results: Optional[list[dict[str, Any]]] = None,
+    error: Optional[str] = None,
+) -> UpdateRunLog:
+    """Write one completed update-run entry.
+
+    Args:
+        run_type: "scheduled" or "manual".
+        trigger: username for manual runs, "scheduler" for automatic ones.
+        pull_results: {image: {"success": bool, "error": str|None}} as returned
+            by image_service.pull_all_images()["results"].
+        customer_results: list of {"customer_name", "success", "error"} as
+            returned by the update-all / update-customer flows.
+    """
+    pull_success = None
+    if pull_results:
+        pull_success = all(r.get("success") for r in pull_results.values())
+
+    customers_total = len(customer_results) if customer_results is not None else None
+    customers_updated = (
+        sum(1 for r in customer_results if r.get("success")) if customer_results is not None else None
+    )
+
+    if error:
+        status = "failed"
+    elif pull_attempted and pull_success is False:
+        status = "failed"
+    elif apply_attempted and customers_total and customers_updated != customers_total:
+        status = "partial"
+    elif not pull_attempted and not customers_total:
+        # Nothing to pull (Hub already matched local) and no customer
+        # container was behind the locally pulled image either.
+        status = "no_update"
+    else:
+        status = "success"
+
+    log = UpdateRunLog(
+        run_type=run_type,
+        trigger=trigger,
+        started_at=started_at,
+        finished_at=datetime.utcnow(),
+        status=status,
+        any_update_available=any_update_available,
+        pull_attempted=pull_attempted,
+        pull_success=pull_success,
+        pull_details=json.dumps(pull_results) if pull_results is not None else None,
+        apply_attempted=apply_attempted,
+        customers_updated=customers_updated,
+        customers_total=customers_total,
+        customer_details=json.dumps(customer_results) if customer_results is not None else None,
+        error=error,
+    )
+    db.add(log)
+    db.commit()
+    return log

app/utils/validators.py

@@ -158,6 +158,27 @@ class SystemConfigUpdate(BaseModel):
     git_repo_url: Optional[str] = Field(None, max_length=500)
     git_branch: Optional[str] = Field(None, max_length=100)
     git_token: Optional[str] = None  # plaintext, encrypted before storage
+    # Automatic NetBird image update check/apply
+    auto_update_check_enabled: Optional[bool] = None
+    auto_update_check_time: Optional[str] = Field(None, max_length=5)
+    auto_update_apply_enabled: Optional[bool] = None
+    # Master default for the NetBird client (peer) automatic-updates feature
+    netbird_client_auto_update_version: Optional[str] = Field(None, max_length=50)
+    netbird_client_auto_update_always: Optional[bool] = None
+    # Automatic restart of unreachable customer containers
+    auto_restart_unreachable_enabled: Optional[bool] = None
+    auto_restart_cooldown_minutes: Optional[int] = Field(None, ge=1, le=1440)
+
+    @field_validator("auto_update_check_time")
+    @classmethod
+    def validate_auto_update_check_time(cls, v: Optional[str]) -> Optional[str]:
+        """Must be HH:MM in 24h format."""
+        if v is None:
+            return v
+        import re
+        if not re.fullmatch(r"([01]\d|2[0-3]):[0-5]\d", v):
+            raise ValueError("auto_update_check_time must be in HH:MM 24h format")
+        return v
 
     @field_validator("ssl_mode")
     @classmethod
@@ -203,6 +224,27 @@ class SystemConfigUpdate(BaseModel):
         return v.lower().strip()
 
 
+# ---------------------------------------------------------------------------
+# NetBird client (peer) automatic updates
+# ---------------------------------------------------------------------------
+class NetbirdClientAutoUpdatePayload(BaseModel):
+    """Push a client auto-update version/mode to one or all customers."""
+
+    version: str = Field(..., max_length=50, description="'latest', 'disabled', or a version e.g. '0.61.0'")
+    always: bool = False
+
+
+class NetbirdApiTokenPayload(BaseModel):
+    """Manually register a NetBird Personal Access Token for a customer.
+
+    Needed for customers deployed before automatic PAT capture existed —
+    create a PAT once in that customer's own NetBird dashboard
+    (Settings > Service Users / Personal Access Tokens) and paste it here.
+    """
+
+    token: str = Field(..., min_length=10, max_length=500)
+
+
 # ---------------------------------------------------------------------------
 # Users
 # ---------------------------------------------------------------------------

static/index.html

@@ -641,6 +641,78 @@
                                             <i class="bi bi-cloud-download me-1"></i><span data-i18n="settings.pullImages">Pull from Docker Hub</span>
                                         </button>
                                         <span id="pull-images-settings-status" class="ms-2 text-muted small"></span>
+                                        <hr>
+                                        <h6 data-i18n="monitoring.autoUpdateTitle">Automatic Updates</h6>
+                                        <p class="text-muted small" data-i18n="monitoring.autoUpdateHint">Automatically checks for new NetBird images daily at the chosen time.</p>
+                                        <form id="settings-auto-update-form">
+                                            <div class="form-check mb-3">
+                                                <input class="form-check-input" type="checkbox" id="cfg-auto-update-check-enabled">
+                                                <label class="form-check-label" for="cfg-auto-update-check-enabled" data-i18n="monitoring.autoUpdateCheckEnabled">Enable automatic update check</label>
+                                            </div>
+                                            <div class="row g-3 align-items-end">
+                                                <div class="col-md-3">
+                                                    <label class="form-label" data-i18n="monitoring.autoUpdateCheckTime">Daily check time</label>
+                                                    <input type="time" class="form-control" id="cfg-auto-update-check-time" value="03:00">
+                                                </div>
+                                            </div>
+                                            <div class="form-check mt-3">
+                                                <input class="form-check-input" type="checkbox" id="cfg-auto-update-apply-enabled">
+                                                <label class="form-check-label" for="cfg-auto-update-apply-enabled" data-i18n="monitoring.autoUpdateApplyEnabled">Automatically update customer containers after check</label>
+                                                <div class="form-text" data-i18n="monitoring.autoUpdateApplyHint">When enabled, all customer containers are automatically recreated after a new image is found (services briefly restart). When disabled, only the images are pulled — updating customers stays a manual step.</div>
+                                            </div>
+                                            <div class="mt-3 small text-muted">
+                                                <span data-i18n="monitoring.autoUpdateLastRun">Last automatic check</span>:
+                                                <span id="auto-update-last-run">-</span>
+                                            </div>
+                                            <div class="mt-3">
+                                                <button type="submit" class="btn btn-primary btn-sm"><i class="bi bi-save me-1"></i><span data-i18n="monitoring.saveAutoUpdateSettings">Save Automation</span></button>
+                                            </div>
+                                        </form>
+                                        <hr>
+                                        <h6 data-i18n="monitoring.autoRestartTitle">Automatic Restart of Unreachable Containers</h6>
+                                        <p class="text-muted small" data-i18n="monitoring.autoRestartHint">Some customers only regain access to their NetBird web UI after a container restart. When enabled, the scheduler periodically checks whether each customer's dashboard actually responds and restarts their containers if it doesn't, logging the action in the customer's log.</p>
+                                        <form id="settings-auto-restart-form">
+                                            <div class="form-check mb-3">
+                                                <input class="form-check-input" type="checkbox" id="cfg-auto-restart-enabled">
+                                                <label class="form-check-label" for="cfg-auto-restart-enabled" data-i18n="monitoring.autoRestartEnabled">Automatically restart unreachable customer containers</label>
+                                            </div>
+                                            <div class="row g-3 align-items-end">
+                                                <div class="col-md-3">
+                                                    <label class="form-label" data-i18n="monitoring.autoRestartCooldown">Cooldown between restarts (minutes)</label>
+                                                    <input type="number" class="form-control" id="cfg-auto-restart-cooldown" min="1" max="1440" value="30">
+                                                </div>
+                                            </div>
+                                            <div class="mt-3">
+                                                <button type="submit" class="btn btn-primary btn-sm"><i class="bi bi-save me-1"></i><span data-i18n="monitoring.saveAutoRestartSettings">Save Automation</span></button>
+                                            </div>
+                                        </form>
+                                        <hr>
+                                        <h6 data-i18n="customer.nbuMasterTitle">NetBird Client Auto-Updates (all customers)</h6>
+                                        <p class="text-muted small" data-i18n="customer.nbuMasterHint">Controls the "Automatic Updates" setting inside every customer's own NetBird dashboard (Settings &gt; Clients). Set the default here, then push it to all customers at once. Individual customers can still be overridden from their detail page.</p>
+                                        <form id="settings-nbu-master-form">
+                                            <div class="row g-2 align-items-end">
+                                                <div class="col-auto">
+                                                    <label class="form-label small mb-1" data-i18n="customer.nbuVersion">Client version</label>
+                                                    <select class="form-select form-select-sm" id="cfg-nbu-version-select" onchange="document.getElementById('cfg-nbu-custom-version').classList.toggle('d-none', this.value !== 'custom')">
+                                                        <option value="disabled" data-i18n="customer.nbuDisabled">Disabled</option>
+                                                        <option value="latest" data-i18n="customer.nbuLatest">Latest</option>
+                                                        <option value="custom" data-i18n="customer.nbuCustom">Specific version</option>
+                                                    </select>
+                                                </div>
+                                                <div class="col-auto">
+                                                    <input type="text" class="form-control form-control-sm d-none" id="cfg-nbu-custom-version" placeholder="0.61.0">
+                                                </div>
+                                                <div class="col-auto form-check pb-1">
+                                                    <input class="form-check-input" type="checkbox" id="cfg-nbu-always">
+                                                    <label class="form-check-label small" for="cfg-nbu-always" data-i18n="customer.nbuForce">Force automatic updates</label>
+                                                </div>
+                                            </div>
+                                            <div class="mt-3">
+                                                <button type="submit" class="btn btn-primary btn-sm me-2"><i class="bi bi-save me-1"></i><span data-i18n="customer.nbuSaveDefault">Save Default</span></button>
+                                                <button type="button" class="btn btn-outline-warning btn-sm" id="btn-nbu-apply-all" onclick="applyNetbirdUpdatesToAll()"><i class="bi bi-broadcast me-1"></i><span data-i18n="customer.nbuApplyAll">Apply to All Customers</span></button>
+                                            </div>
+                                        </form>
+                                        <div id="nbu-apply-all-result" class="mt-3"></div>
                                     </div>
                                 </div>
                             </div>
@@ -1206,6 +1278,19 @@
                         <p class="text-muted mb-0" data-i18n="monitoring.clickCheckUpdates">Click "Check for Updates" to compare local images with Docker Hub.</p>
                     </div>
                 </div>
+
+                <!-- Update Run Log -->
+                <div class="card shadow-sm mt-4">
+                    <div class="card-header d-flex justify-content-between align-items-center">
+                        <span><i class="bi bi-clock-history me-2"></i><span data-i18n="monitoring.updateLogTitle">Update Log</span></span>
+                        <button class="btn btn-outline-secondary btn-sm" onclick="loadUpdateLog()" id="btn-refresh-update-log">
+                            <i class="bi bi-arrow-clockwise me-1"></i><span data-i18n="monitoring.refresh">Refresh</span>
+                        </button>
+                    </div>
+                    <div class="card-body" id="update-log-body">
+                        <p class="text-muted mb-0" data-i18n="common.loading">Loading...</p>
+                    </div>
+                </div>
             </div>
         </div>
     </div>

static/js/app.js

@@ -571,6 +571,146 @@ function goToPage(page) {
     loadCustomers();
 }
 
+// ---------------------------------------------------------------------------
+// NetBird client (peer) automatic-updates — per-customer
+// ---------------------------------------------------------------------------
+function _nbuVersionOptions(selected) {
+    const opts = [
+        ['disabled', t('customer.nbuDisabled')],
+        ['latest', t('customer.nbuLatest')],
+        ['custom', t('customer.nbuCustom')],
+    ];
+    const isCustom = selected && selected !== 'disabled' && selected !== 'latest';
+    return opts.map(([v, label]) =>
+        `<option value="${v}" ${(!isCustom && v === selected) || (isCustom && v === 'custom') ? 'selected' : ''}>${label}</option>`
+    ).join('');
+}
+
+async function loadCustomerNetbirdUpdates(id, hasToken) {
+    const container = document.getElementById('nbu-container');
+    if (!container) return;
+
+    if (!hasToken) {
+        container.innerHTML = `
+            <p class="text-muted small mb-2">${t('customer.nbuNoToken')}</p>
+            <div class="input-group input-group-sm">
+                <input type="text" class="form-control" id="nbu-token-input" placeholder="${t('customer.nbuTokenPlaceholder')}">
+                <button class="btn btn-outline-primary" onclick="saveCustomerNetbirdToken(${id})">${t('customer.nbuSaveToken')}</button>
+            </div>
+            <div id="nbu-token-result" class="small mt-1"></div>`;
+        return;
+    }
+
+    container.innerHTML = `<span class="spinner-border spinner-border-sm"></span>`;
+    try {
+        const data = await api('GET', `/customers/${id}/netbird-updates`);
+        const isCustom = data.version && data.version !== 'disabled' && data.version !== 'latest';
+
+        let tokenInfoHtml = '';
+        if (data.token_renewed_at) {
+            const renewedDate = new Date(data.token_renewed_at);
+            const expiresDate = new Date(renewedDate);
+            expiresDate.setDate(expiresDate.getDate() + 365);
+            tokenInfoHtml = `<div class="small text-muted mb-2">${t('customer.nbuTokenRenewed', { date: renewedDate.toLocaleDateString() })} &middot; ${t('customer.nbuTokenExpires', { date: expiresDate.toLocaleDateString() })}</div>`;
+        }
+
+        container.innerHTML = `
+            ${tokenInfoHtml}
+            <div class="row g-2 align-items-end">
+                <div class="col-auto">
+                    <label class="form-label small mb-1">${t('customer.nbuVersion')}</label>
+                    <select class="form-select form-select-sm" id="nbu-version-select" onchange="document.getElementById('nbu-custom-version').classList.toggle('d-none', this.value !== 'custom')">
+                        ${_nbuVersionOptions(data.version)}
+                    </select>
+                </div>
+                <div class="col-auto">
+                    <input type="text" class="form-control form-control-sm ${isCustom ? '' : 'd-none'}" id="nbu-custom-version" placeholder="0.61.0" value="${isCustom ? esc(data.version) : ''}">
+                </div>
+                <div class="col-auto form-check pb-1">
+                    <input class="form-check-input" type="checkbox" id="nbu-always" ${data.always ? 'checked' : ''}>
+                    <label class="form-check-label small" for="nbu-always">${t('customer.nbuForce')}</label>
+                </div>
+                <div class="col-auto">
+                    <button class="btn btn-primary btn-sm" onclick="saveCustomerNetbirdUpdate(${id})">${t('customer.nbuSave')}</button>
+                    <button class="btn btn-outline-secondary btn-sm" onclick="syncCustomerNetbirdFromMaster(${id})">${t('customer.nbuSyncMaster')}</button>
+                </div>
+            </div>
+            <div id="nbu-result" class="small mt-2"></div>
+            <div class="mt-2">
+                <a href="#" class="small" onclick="event.preventDefault(); showChangeNetbirdToken(${id})">${t('customer.nbuChangeToken')}</a>
+            </div>
+            <div id="nbu-change-token-area" class="mt-2 d-none"></div>`;
+    } catch (err) {
+        container.innerHTML = `<div class="alert alert-warning py-2 small mb-0">${esc(err.message)}</div>`;
+    }
+}
+
+function showChangeNetbirdToken(id) {
+    const area = document.getElementById('nbu-change-token-area');
+    area.classList.remove('d-none');
+    area.innerHTML = `
+        <div class="input-group input-group-sm">
+            <input type="text" class="form-control" id="nbu-token-input" placeholder="${t('customer.nbuTokenPlaceholder')}">
+            <button class="btn btn-outline-primary" onclick="saveCustomerNetbirdToken(${id})">${t('customer.nbuSaveToken')}</button>
+        </div>
+        <div id="nbu-token-result" class="small mt-1"></div>`;
+}
+
+async function saveCustomerNetbirdToken(id) {
+    const input = document.getElementById('nbu-token-input');
+    const resultEl = document.getElementById('nbu-token-result');
+    const token = input.value.trim();
+    if (!token) return;
+    resultEl.innerHTML = `<span class="spinner-border spinner-border-sm"></span>`;
+    try {
+        await api('PUT', `/customers/${id}/netbird-api-token`, { token });
+        showToast(t('customer.nbuTokenSaved'));
+        loadCustomerNetbirdUpdates(id, true);
+    } catch (err) {
+        resultEl.innerHTML = `<span class="text-danger">${esc(err.message)}</span>`;
+    }
+}
+
+async function _readNbuForm() {
+    const select = document.getElementById('nbu-version-select').value;
+    const version = select === 'custom' ? document.getElementById('nbu-custom-version').value.trim() : select;
+    const always = document.getElementById('nbu-always').checked;
+    return { version, always };
+}
+
+async function saveCustomerNetbirdUpdate(id) {
+    const resultEl = document.getElementById('nbu-result');
+    const payload = await _readNbuForm();
+    if (!payload.version) {
+        resultEl.innerHTML = `<span class="text-danger">${t('customer.nbuVersionRequired')}</span>`;
+        return;
+    }
+    resultEl.innerHTML = `<span class="spinner-border spinner-border-sm"></span>`;
+    try {
+        await api('PUT', `/customers/${id}/netbird-updates`, payload);
+        showToast(t('customer.nbuSaved'));
+        loadCustomerNetbirdUpdates(id, true);
+    } catch (err) {
+        resultEl.innerHTML = `<span class="text-danger">${esc(err.message)}</span>`;
+    }
+}
+
+async function syncCustomerNetbirdFromMaster(id) {
+    const resultEl = document.getElementById('nbu-result');
+    resultEl.innerHTML = `<span class="spinner-border spinner-border-sm"></span>`;
+    try {
+        const cfg = await api('GET', '/settings/system');
+        await api('PUT', `/customers/${id}/netbird-updates`, {
+            version: cfg.netbird_client_auto_update_version,
+            always: cfg.netbird_client_auto_update_always,
+        });
+        showToast(t('customer.nbuSynced'));
+        loadCustomerNetbirdUpdates(id, true);
+    } catch (err) {
+        resultEl.innerHTML = `<span class="text-danger">${esc(err.message)}</span>`;
+    }
+}
+
 // Search & filter listeners
 document.getElementById('search-input').addEventListener('input', debounce(() => { customersPage = 1; loadCustomers(); }, 300));
 document.getElementById('status-filter').addEventListener('change', () => { customersPage = 1; loadCustomers(); });
@@ -812,6 +952,14 @@ async function viewCustomer(id) {
                         ` : `<p class="text-muted mb-0">${t('customer.credentialsNotAvailable')}</p>`}
                     </div>
                 </div>
+                <div class="card mt-3">
+                    <div class="card-header">
+                        <strong><i class="bi bi-phone me-1"></i>${t('customer.netbirdClientUpdates')}</strong>
+                    </div>
+                    <div class="card-body" id="nbu-container">
+                        <span class="spinner-border spinner-border-sm"></span>
+                    </div>
+                </div>
                 <div class="mt-3">
                     <button class="btn btn-success btn-sm me-1" onclick="customerAction(${id},'start')"><i class="bi bi-play-circle me-1"></i>${t('customer.start')}</button>
                     <button class="btn btn-warning btn-sm me-1" onclick="customerAction(${id},'stop')"><i class="bi bi-stop-circle me-1"></i>${t('customer.stop')}</button>
@@ -824,6 +972,7 @@ async function viewCustomer(id) {
                 </div>
                 <div id="detail-update-result"></div>
             `;
+            loadCustomerNetbirdUpdates(id, d.has_netbird_api_token);
         } else {
             document.getElementById('detail-deployment-content').innerHTML = `
                 <p class="text-muted">${t('customer.noDeployment')}</p>
@@ -940,6 +1089,23 @@ async function loadSettings() {
         document.getElementById('cfg-relay-image').value = cfg.netbird_relay_image || '';
         document.getElementById('cfg-dashboard-image').value = cfg.netbird_dashboard_image || '';
 
+        document.getElementById('cfg-auto-update-check-enabled').checked = cfg.auto_update_check_enabled || false;
+        document.getElementById('cfg-auto-update-check-time').value = cfg.auto_update_check_time || '03:00';
+        document.getElementById('cfg-auto-update-apply-enabled').checked = cfg.auto_update_apply_enabled || false;
+        document.getElementById('auto-update-last-run').textContent = cfg.auto_update_last_run_at
+            ? new Date(cfg.auto_update_last_run_at).toLocaleString()
+            : t('monitoring.autoUpdateNever');
+
+        document.getElementById('cfg-auto-restart-enabled').checked = cfg.auto_restart_unreachable_enabled || false;
+        document.getElementById('cfg-auto-restart-cooldown').value = cfg.auto_restart_cooldown_minutes || 30;
+
+        const nbuVersion = cfg.netbird_client_auto_update_version || 'disabled';
+        const nbuIsCustom = nbuVersion !== 'disabled' && nbuVersion !== 'latest';
+        document.getElementById('cfg-nbu-version-select').value = nbuIsCustom ? 'custom' : nbuVersion;
+        document.getElementById('cfg-nbu-custom-version').value = nbuIsCustom ? nbuVersion : '';
+        document.getElementById('cfg-nbu-custom-version').classList.toggle('d-none', !nbuIsCustom);
+        document.getElementById('cfg-nbu-always').checked = cfg.netbird_client_auto_update_always || false;
+
         // Branding tab
         document.getElementById('cfg-branding-name').value = cfg.branding_name || '';
         document.getElementById('cfg-branding-subtitle').value = cfg.branding_subtitle || '';
@@ -1058,6 +1224,96 @@ document.getElementById('settings-images-form').addEventListener('submit', async
     }
 });
 
+// Automatic update settings form
+document.getElementById('settings-auto-update-form').addEventListener('submit', async (e) => {
+    e.preventDefault();
+    try {
+        await api('PUT', '/settings/system', {
+            auto_update_check_enabled: document.getElementById('cfg-auto-update-check-enabled').checked,
+            auto_update_check_time: document.getElementById('cfg-auto-update-check-time').value || '03:00',
+            auto_update_apply_enabled: document.getElementById('cfg-auto-update-apply-enabled').checked,
+        });
+        showSettingsAlert('success', t('messages.imageSettingsSaved'));
+    } catch (err) {
+        showSettingsAlert('danger', t('errors.failed', { error: err.message }));
+    }
+});
+
+// Automatic restart of unreachable customer containers
+document.getElementById('settings-auto-restart-form').addEventListener('submit', async (e) => {
+    e.preventDefault();
+    try {
+        await api('PUT', '/settings/system', {
+            auto_restart_unreachable_enabled: document.getElementById('cfg-auto-restart-enabled').checked,
+            auto_restart_cooldown_minutes: parseInt(document.getElementById('cfg-auto-restart-cooldown').value, 10) || 30,
+        });
+        showSettingsAlert('success', t('messages.imageSettingsSaved'));
+    } catch (err) {
+        showSettingsAlert('danger', t('errors.failed', { error: err.message }));
+    }
+});
+
+function _readNbuMasterForm() {
+    const select = document.getElementById('cfg-nbu-version-select').value;
+    const version = select === 'custom' ? document.getElementById('cfg-nbu-custom-version').value.trim() : select;
+    const always = document.getElementById('cfg-nbu-always').checked;
+    return { version, always };
+}
+
+// NetBird client auto-update master default form
+document.getElementById('settings-nbu-master-form').addEventListener('submit', async (e) => {
+    e.preventDefault();
+    const { version, always } = _readNbuMasterForm();
+    if (!version) {
+        showSettingsAlert('danger', t('customer.nbuVersionRequired'));
+        return;
+    }
+    try {
+        await api('PUT', '/settings/system', {
+            netbird_client_auto_update_version: version,
+            netbird_client_auto_update_always: always,
+        });
+        showSettingsAlert('success', t('messages.imageSettingsSaved'));
+    } catch (err) {
+        showSettingsAlert('danger', t('errors.failed', { error: err.message }));
+    }
+});
+
+async function applyNetbirdUpdatesToAll() {
+    const { version, always } = _readNbuMasterForm();
+    if (!version) {
+        showSettingsAlert('danger', t('customer.nbuVersionRequired'));
+        return;
+    }
+    if (!confirm(t('customer.nbuConfirmApplyAll'))) return;
+
+    const btn = document.getElementById('btn-nbu-apply-all');
+    const resultDiv = document.getElementById('nbu-apply-all-result');
+    btn.disabled = true;
+    resultDiv.innerHTML = `<span class="spinner-border spinner-border-sm me-2"></span>${t('common.loading')}`;
+    try {
+        const data = await api('POST', '/monitoring/netbird-updates/apply-all', { version, always });
+        const rows = data.results.map(r => `<tr>
+            <td>${esc(r.customer_name)}</td>
+            <td>${r.success
+                ? '<span class="badge bg-success"><i class="bi bi-check-lg"></i> OK</span>'
+                : '<span class="badge bg-danger"><i class="bi bi-x-lg"></i> Error</span>'}</td>
+            <td class="small text-muted">${esc(r.error || '')}</td>
+        </tr>`).join('');
+        resultDiv.innerHTML = `<div class="alert alert-${data.updated === data.results.length ? 'success' : 'warning'}">
+            <strong>${esc(data.message)}</strong>
+            <table class="table table-sm mb-0 mt-2">
+                <thead><tr><th>${t('monitoring.thName')}</th><th>${t('monitoring.thStatus')}</th><th></th></tr></thead>
+                <tbody>${rows}</tbody>
+            </table>
+        </div>`;
+    } catch (err) {
+        resultDiv.innerHTML = `<div class="alert alert-danger">${esc(err.message)}</div>`;
+    } finally {
+        btn.disabled = false;
+    }
+}
+
 // Test NPM connection
 async function testNpmConnection() {
     const spinner = document.getElementById('npm-test-spinner');
@@ -1704,7 +1960,90 @@ function togglePasswordVisibility(inputId) {
 // Monitoring
 // ---------------------------------------------------------------------------
 async function loadMonitoring() {
-    await Promise.all([loadResources(), loadAllCustomerStatuses()]);
+    await Promise.all([loadResources(), loadAllCustomerStatuses(), loadUpdateLog()]);
+}
+
+const _updateLogStatusBadge = {
+    success: 'bg-success',
+    no_update: 'bg-secondary',
+    partial: 'bg-warning text-dark',
+    failed: 'bg-danger',
+    running: 'bg-info',
+};
+
+async function loadUpdateLog() {
+    const body = document.getElementById('update-log-body');
+    body.innerHTML = `<span class="spinner-border spinner-border-sm me-2"></span>${t('common.loading')}`;
+    try {
+        const logs = await api('GET', '/monitoring/update-logs?limit=30');
+        if (logs.length === 0) {
+            body.innerHTML = `<p class="text-muted mb-0">${t('monitoring.noUpdateLogs')}</p>`;
+            return;
+        }
+        const rows = logs.map((l, i) => {
+            const badgeClass = _updateLogStatusBadge[l.status] || 'bg-secondary';
+            const typeLabel = l.run_type === 'scheduled' ? t('monitoring.logTypeScheduled') : t('monitoring.logTypeManual');
+            const pullSummary = l.pull_attempted
+                ? (l.pull_success ? t('monitoring.logPullOk') : t('monitoring.logPullFailed'))
+                : t('monitoring.logPullSkipped');
+            const applySummary = l.apply_attempted
+                ? t('monitoring.logApplySummary', { updated: l.customers_updated ?? 0, total: l.customers_total ?? 0 })
+                : t('monitoring.logApplySkipped');
+            const detailId = `update-log-detail-${i}`;
+            const hasDetails = (l.pull_details && Object.keys(l.pull_details).length > 0) ||
+                (l.customer_details && l.customer_details.length > 0) || l.error;
+            const detailsHtml = hasDetails ? `
+                <div class="collapse mt-2" id="${detailId}">
+                    ${l.error ? `<div class="alert alert-danger py-2 small mb-2">${esc(l.error)}</div>` : ''}
+                    ${l.pull_details ? `
+                        <table class="table table-sm mb-2">
+                            <thead><tr><th>${t('monitoring.thImage')}</th><th>${t('monitoring.thStatus')}</th><th></th></tr></thead>
+                            <tbody>${Object.entries(l.pull_details).map(([img, r]) => `<tr>
+                                <td><code class="small">${esc(img)}</code></td>
+                                <td>${r.success ? `<span class="badge bg-success">OK</span>` : `<span class="badge bg-danger">Error</span>`}</td>
+                                <td class="small text-muted">${esc(r.error || '')}</td>
+                            </tr>`).join('')}</tbody>
+                        </table>` : ''}
+                    ${l.customer_details && l.customer_details.length > 0 ? `
+                        <table class="table table-sm mb-0">
+                            <thead><tr><th>${t('monitoring.thName')}</th><th>${t('monitoring.thStatus')}</th><th></th></tr></thead>
+                            <tbody>${l.customer_details.map(r => `<tr>
+                                <td>${esc(r.customer_name)}</td>
+                                <td>${r.success ? `<span class="badge bg-success">OK</span>` : `<span class="badge bg-danger">Error</span>`}</td>
+                                <td class="small text-muted">${esc(r.error || '')}</td>
+                            </tr>`).join('')}</tbody>
+                        </table>` : ''}
+                </div>` : '';
+            return `<tr>
+                <td class="small text-nowrap">${formatDate(l.started_at)}</td>
+                <td><span class="badge bg-light text-dark border">${typeLabel}</span></td>
+                <td><span class="badge ${badgeClass}">${t('monitoring.logStatus_' + l.status)}</span></td>
+                <td class="small">${pullSummary}</td>
+                <td class="small">${applySummary}</td>
+                <td class="small text-muted">${esc(l.trigger || '')}</td>
+                <td>${hasDetails ? `<button class="btn btn-sm btn-outline-secondary" data-bs-toggle="collapse" data-bs-target="#${detailId}"><i class="bi bi-chevron-down"></i></button>` : ''}</td>
+            </tr>${detailsHtml ? `<tr class="d-none"></tr><tr><td colspan="7" class="p-0 border-0">${detailsHtml}</td></tr>` : ''}`;
+        }).join('');
+        body.innerHTML = `
+            <div class="table-responsive">
+                <table class="table table-sm mb-0">
+                    <thead class="table-light">
+                        <tr>
+                            <th>${t('monitoring.logTime')}</th>
+                            <th>${t('monitoring.logType')}</th>
+                            <th>${t('monitoring.thStatus')}</th>
+                            <th>${t('monitoring.logPull')}</th>
+                            <th>${t('monitoring.logApply')}</th>
+                            <th>${t('monitoring.logTrigger')}</th>
+                            <th></th>
+                        </tr>
+                    </thead>
+                    <tbody>${rows}</tbody>
+                </table>
+            </div>`;
+    } catch (err) {
+        body.innerHTML = `<div class="alert alert-danger">${esc(err.message)}</div>`;
+    }
 }
 
 async function loadResources() {
@@ -1807,6 +2146,8 @@ async function checkImageUpdates() {
             : data.customer_status.map(c => {
                 const badge = c.needs_update
                     ? `<span class="badge bg-warning text-dark">${t('monitoring.needsUpdate')}</span>`
+                    : c.unknown
+                        ? `<span class="badge bg-secondary" title="${t('monitoring.statusUnknownHint')}">${t('monitoring.statusUnknown')}</span>`
                         : `<span class="badge bg-success">${t('monitoring.upToDate')}</span>`;
                 const updateBtn = c.needs_update
                     ? `<button class="btn btn-sm btn-outline-warning ms-2 btn-update-customer" onclick="updateCustomerImages(${c.customer_id})"

static/lang/de.json

@@ -91,7 +91,30 @@
     "lastCheck": "Letzte Prüfung: {time}",
     "openDashboard": "Dashboard öffnen",
     "updateImages": "Images aktualisieren",
-    "updateInProgress": "Container werden aktualisiert — bitte warten…"
+    "updateInProgress": "Container werden aktualisiert — bitte warten…",
+    "netbirdClientUpdates": "NetBird Client Auto-Updates",
+    "nbuNoToken": "Kein API-Token für diesen Kunden hinterlegt. Bei Neu-Deployments wird das automatisch erfasst — für bestehende Kunden einmalig ein Personal Access Token im Kunden-Dashboard erstellen (Settings → Service Users) und hier einfügen.",
+    "nbuTokenPlaceholder": "Personal Access Token einfügen…",
+    "nbuSaveToken": "Prüfen & Speichern",
+    "nbuTokenSaved": "Token gespeichert.",
+    "nbuChangeToken": "Token ändern",
+    "nbuTokenRenewed": "Token erneuert am {date}",
+    "nbuTokenExpires": "gültig bis {date}",
+    "nbuVersion": "Client-Version",
+    "nbuDisabled": "Deaktiviert",
+    "nbuLatest": "Neueste Version",
+    "nbuCustom": "Bestimmte Version",
+    "nbuForce": "Automatische Updates erzwingen",
+    "nbuSave": "Speichern",
+    "nbuSyncMaster": "Vom Standard übernehmen",
+    "nbuSaved": "Einstellung übernommen.",
+    "nbuSynced": "Standard-Einstellung übernommen.",
+    "nbuVersionRequired": "Bitte eine Version angeben.",
+    "nbuMasterTitle": "NetBird Client Auto-Updates (alle Kunden)",
+    "nbuMasterHint": "Steuert die \"Automatische Updates\"-Einstellung im NetBird-Dashboard jedes Kunden (Settings → Clients). Hier den Standard festlegen und auf alle Kunden anwenden. Einzelne Kunden können weiterhin über ihre Detailseite abweichend eingestellt werden.",
+    "nbuSaveDefault": "Standard speichern",
+    "nbuApplyAll": "Auf alle Kunden anwenden",
+    "nbuConfirmApplyAll": "Diese Update-Einstellung auf alle Kunden mit hinterlegtem API-Token anwenden?"
   },
   "settings": {
     "title": "Systemeinstellungen",
@@ -420,6 +443,41 @@
     "updating": "Wird aktualisiert…",
     "updateAllProgress": "Kunden-Container werden nacheinander aktualisiert — bitte warten…",
     "pulling": "Wird geladen…",
-    "pullStartedShort": "Download im Hintergrund gestartet."
+    "pullStartedShort": "Download im Hintergrund gestartet.",
+    "statusUnknown": "Unbekannt",
+    "statusUnknownHint": "Container konnte nicht eindeutig zugeordnet werden (z. B. nicht gestartet). Kein verifiziertes \"Aktuell\".",
+    "autoUpdateTitle": "Automatische Aktualisierung",
+    "autoUpdateHint": "Prüft täglich zur gewählten Uhrzeit automatisch auf neue NetBird-Images.",
+    "autoUpdateCheckEnabled": "Automatische Update-Prüfung aktivieren",
+    "autoUpdateCheckTime": "Uhrzeit der täglichen Prüfung",
+    "autoUpdateApplyEnabled": "Kunden-Container nach Prüfung automatisch aktualisieren",
+    "autoUpdateApplyHint": "Wenn aktiviert, werden nach einer gefundenen Aktualisierung automatisch alle Kunden-Container neu erstellt (kurzer Neustart der Dienste). Wenn deaktiviert, werden nur die Images geladen — die Aktualisierung der Kunden erfolgt weiterhin manuell.",
+    "autoUpdateLastRun": "Letzte automatische Prüfung",
+    "autoUpdateNever": "Noch nie ausgeführt",
+    "saveAutoUpdateSettings": "Automatisierung speichern",
+    "autoRestartTitle": "Automatischer Neustart nicht erreichbarer Container",
+    "autoRestartHint": "Manche Kunden erhalten den Zugriff auf ihre NetBird-Weboberfläche erst nach einem Container-Neustart zurück. Wenn aktiviert, prüft der Scheduler regelmäßig, ob die Weboberfläche jedes Kunden tatsächlich antwortet, und startet die Container bei Bedarf neu — mit Protokollierung im Kunden-Log.",
+    "autoRestartEnabled": "Nicht erreichbare Kunden-Container automatisch neu starten",
+    "autoRestartCooldown": "Wartezeit zwischen Neustarts (Minuten)",
+    "saveAutoRestartSettings": "Automatisierung speichern",
+    "updateLogTitle": "Update-Verlauf",
+    "noUpdateLogs": "Noch keine Update-Läufe protokolliert.",
+    "logTypeScheduled": "Automatisch",
+    "logTypeManual": "Manuell",
+    "logTime": "Zeitpunkt",
+    "logType": "Typ",
+    "logPull": "Image-Download",
+    "logApply": "Kunden-Update",
+    "logTrigger": "Ausgelöst von",
+    "logPullOk": "Erfolgreich",
+    "logPullFailed": "Fehlgeschlagen",
+    "logPullSkipped": "Nicht nötig",
+    "logApplySummary": "{updated}/{total} Kunden",
+    "logApplySkipped": "Nicht ausgeführt",
+    "logStatus_success": "Erfolgreich",
+    "logStatus_no_update": "Kein Update nötig",
+    "logStatus_partial": "Teilweise",
+    "logStatus_failed": "Fehlgeschlagen",
+    "logStatus_running": "Läuft…"
   }
 }

static/lang/en.json

@@ -91,7 +91,30 @@
     "lastCheck": "Last check: {time}",
     "openDashboard": "Open Dashboard",
     "updateImages": "Update Images",
-    "updateInProgress": "Updating containers — please wait…"
+    "updateInProgress": "Updating containers — please wait…",
+    "netbirdClientUpdates": "NetBird Client Auto-Updates",
+    "nbuNoToken": "No API token registered for this customer. New deployments capture one automatically — for existing customers, create a Personal Access Token once in their dashboard (Settings → Service Users) and paste it here.",
+    "nbuTokenPlaceholder": "Paste Personal Access Token…",
+    "nbuSaveToken": "Verify & Save",
+    "nbuTokenSaved": "Token saved.",
+    "nbuChangeToken": "Change token",
+    "nbuTokenRenewed": "Token renewed on {date}",
+    "nbuTokenExpires": "valid until {date}",
+    "nbuVersion": "Client version",
+    "nbuDisabled": "Disabled",
+    "nbuLatest": "Latest version",
+    "nbuCustom": "Specific version",
+    "nbuForce": "Force automatic updates",
+    "nbuSave": "Save",
+    "nbuSyncMaster": "Sync from default",
+    "nbuSaved": "Setting applied.",
+    "nbuSynced": "Default setting applied.",
+    "nbuVersionRequired": "Please specify a version.",
+    "nbuMasterTitle": "NetBird Client Auto-Updates (all customers)",
+    "nbuMasterHint": "Controls the \"Automatic Updates\" setting inside every customer's own NetBird dashboard (Settings → Clients). Set the default here, then push it to all customers at once. Individual customers can still be overridden from their detail page.",
+    "nbuSaveDefault": "Save Default",
+    "nbuApplyAll": "Apply to All Customers",
+    "nbuConfirmApplyAll": "Apply this update setting to every customer with a registered API token?"
   },
   "customerModal": {
     "newCustomer": "New Customer",
@@ -327,7 +350,42 @@
     "updating": "Updating…",
     "updateAllProgress": "Updating customer containers one by one — please wait…",
     "pulling": "Pulling…",
-    "pullStartedShort": "Pull started in background."
+    "pullStartedShort": "Pull started in background.",
+    "statusUnknown": "Unknown",
+    "statusUnknownHint": "Container could not be matched reliably (e.g. not running). Not a verified \"up to date\".",
+    "autoUpdateTitle": "Automatic Updates",
+    "autoUpdateHint": "Automatically checks for new NetBird images daily at the chosen time.",
+    "autoUpdateCheckEnabled": "Enable automatic update check",
+    "autoUpdateCheckTime": "Daily check time",
+    "autoUpdateApplyEnabled": "Automatically update customer containers after check",
+    "autoUpdateApplyHint": "When enabled, all customer containers are automatically recreated after a new image is found (services briefly restart). When disabled, only the images are pulled — updating customers stays a manual step.",
+    "autoUpdateLastRun": "Last automatic check",
+    "autoUpdateNever": "Never run",
+    "saveAutoUpdateSettings": "Save Automation",
+    "autoRestartTitle": "Automatic Restart of Unreachable Containers",
+    "autoRestartHint": "Some customers only regain access to their NetBird web UI after a container restart. When enabled, the scheduler periodically checks whether each customer's dashboard actually responds and restarts their containers if it doesn't, logging the action in the customer's log.",
+    "autoRestartEnabled": "Automatically restart unreachable customer containers",
+    "autoRestartCooldown": "Cooldown between restarts (minutes)",
+    "saveAutoRestartSettings": "Save Automation",
+    "updateLogTitle": "Update Log",
+    "noUpdateLogs": "No update runs logged yet.",
+    "logTypeScheduled": "Automatic",
+    "logTypeManual": "Manual",
+    "logTime": "Time",
+    "logType": "Type",
+    "logPull": "Image Pull",
+    "logApply": "Customer Update",
+    "logTrigger": "Triggered by",
+    "logPullOk": "Succeeded",
+    "logPullFailed": "Failed",
+    "logPullSkipped": "Not needed",
+    "logApplySummary": "{updated}/{total} customers",
+    "logApplySkipped": "Not run",
+    "logStatus_success": "Success",
+    "logStatus_no_update": "No update needed",
+    "logStatus_partial": "Partial",
+    "logStatus_failed": "Failed",
+    "logStatus_running": "Running…"
   },
   "userModal": {
     "title": "New User",

templates/docker-compose.yml.j2

@@ -20,6 +20,11 @@ services:
     image: {{ netbird_management_image }}
     container_name: netbird-{{ subdomain }}-management
     restart: unless-stopped
+    environment:
+      # Allows the MSP appliance to request a Personal Access Token during the
+      # one-time /api/setup bootstrap call. The endpoint itself locks down
+      # (412) as soon as the first user exists, so leaving this on is safe.
+      - NB_SETUP_PAT_ENABLED=true
     networks:
       - {{ docker_network }}
     volumes: