perf(monitoring): stop blocking event loop with synchronous Docker calls

Customer search and detail loads were intermittently slow because every
customer-table render (including each search keystroke) triggered
/monitoring/customers/local-update-status, which looped synchronously over
all customers doing blocking `docker inspect` subprocess calls on the event
loop — stalling all other in-flight requests, including search itself.

- Offload per-service image/container inspection to the thread pool and run
  checks concurrently instead of sequentially (image_service, docker_service)
- Reuse a single Docker SDK client instead of reconnecting per customer
- Cache local-update-status results for 20s since the underlying data only
  changes after an image pull, not on every keystroke
- Parallelize /monitoring/customers container status lookups

Co-Authored-By: Claude Sonnet 5 <[email protected]>

app/main.py

@@ -33,7 +33,7 @@ logger = logging.getLogger(__name__)
 app = FastAPI(
     title="NetBird MSP Appliance",
     description="Multi-tenant NetBird management platform for MSPs",
-    version="1.0.0",
+    version="1.1.0",
     docs_url="/api/docs",
     redoc_url="/api/redoc",
     openapi_url="/api/openapi.json",
@@ -90,16 +90,36 @@ STATIC_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "static")
 if os.path.isdir(STATIC_DIR):
     app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
 
-# Serve index.html at root
-from fastapi.responses import FileResponse
+# Serve index.html at root — inject cache-busting version into static asset URLs
+# so the browser always loads fresh JS/CSS after a container update.
+from fastapi.responses import FileResponse, HTMLResponse
+from app.services import update_service
+
+_STATIC_ASSETS = (
+    '"/static/js/app.js"',
+    '"/static/js/i18n.js"',
+    '"/static/css/styles.css"',
+)
+
+def _cache_bust_index(html: str, version: str) -> str:
+    # Inject version as a global JS variable so i18n.js can bust lang file caches too
+    html = html.replace("</head>", f'<script>window.STATIC_VERSION="{version}";</script>\n</head>', 1)
+    for asset in _STATIC_ASSETS:
+        busted = asset.rstrip('"') + f'?v={version}"'
+        html = html.replace(asset, busted)
+    return html
+
 
 @app.get("/", include_in_schema=False)
 async def serve_index():
-    """Serve the main dashboard."""
+    """Serve the main dashboard with cache-busted static asset URLs."""
     index_path = os.path.join(STATIC_DIR, "index.html")
-    if os.path.isfile(index_path):
-        return FileResponse(index_path)
+    if not os.path.isfile(index_path):
         return JSONResponse({"message": "NetBird MSP Appliance API is running."})
+    version = update_service.get_current_version().get("commit", "unknown")
+    html = open(index_path, encoding="utf-8").read()
+    html = _cache_bust_index(html, version)
+    return HTMLResponse(content=html, headers={"Cache-Control": "no-cache"})
 
 
 # ---------------------------------------------------------------------------

app/routers/deployments.py

@@ -2,7 +2,7 @@
 
 import logging
 
-from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, status
+from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query, status
 from sqlalchemy.orm import Session
 
 from app.database import SessionLocal, get_db
@@ -19,6 +19,14 @@ router = APIRouter()
 async def manual_deploy(
     customer_id: int,
     background_tasks: BackgroundTasks,
+    keep_data: bool = Query(
+        False,
+        description=(
+            "If True, preserve existing NetBird data (database, keys, peers). "
+            "Containers are recreated without wiping the instance directory. "
+            "If False (default), the instance is fully removed and redeployed from scratch."
+        ),
+    ),
     current_user: User = Depends(get_current_user),
     db: Session = Depends(get_db),
 ):
@@ -29,6 +37,7 @@ async def manual_deploy(
 
     Args:
         customer_id: Customer ID.
+        keep_data: Whether to preserve existing NetBird data.
 
     Returns:
         Acknowledgement dict.
@@ -40,12 +49,12 @@ async def manual_deploy(
     customer.status = "deploying"
     db.commit()
 
-    async def _deploy_bg(cid: int) -> None:
+    async def _deploy_bg(cid: int, keep: bool) -> None:
         bg_db = SessionLocal()
         try:
-            # Remove existing deployment if present
             existing = bg_db.query(Deployment).filter(Deployment.customer_id == cid).first()
-            if existing:
+            if existing and not keep:
+                # Full redeploy: remove everything first
                 await netbird_service.undeploy_customer(bg_db, cid)
             await netbird_service.deploy_customer(bg_db, cid)
         except Exception:
@@ -53,7 +62,7 @@ async def manual_deploy(
         finally:
             bg_db.close()
 
-    background_tasks.add_task(_deploy_bg, customer_id)
+    background_tasks.add_task(_deploy_bg, customer_id, keep_data)
     return {"message": "Deployment started in background.", "status": "deploying"}
 
 

app/routers/monitoring.py

@@ -1,7 +1,9 @@
 """Monitoring API — system overview, customer statuses, host resources."""
 
+import asyncio
 import logging
 import platform
+import time
 from typing import Any
 
 import psutil
@@ -16,6 +18,13 @@ from app.services import docker_service, image_service
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
+# Short-lived cache for the local update-status badges. This endpoint is
+# triggered on every customer-table render (i.e. every search keystroke), but
+# the underlying data (which images are outdated) only changes after an image
+# pull + container recreate, so a few seconds of staleness is harmless.
+_update_status_cache: dict[str, Any] = {"data": None, "expires": 0.0}
+_UPDATE_STATUS_TTL_SECONDS = 20
+
 
 @router.get("/status")
 async def system_status(
@@ -58,8 +67,7 @@ async def all_customers_status(
         .all()
     )
 
-    results: list[dict[str, Any]] = []
-    for c in customers:
+    async def _build_entry(c: Customer) -> dict[str, Any]:
         entry: dict[str, Any] = {
             "id": c.id,
             "name": c.name,
@@ -67,7 +75,7 @@ async def all_customers_status(
             "status": c.status,
         }
         if c.deployment:
-            containers = docker_service.get_container_status(c.deployment.container_prefix)
+            containers = await docker_service.get_container_status_async(c.deployment.container_prefix)
             entry["deployment_status"] = c.deployment.deployment_status
             entry["containers"] = containers
             entry["relay_udp_port"] = c.deployment.relay_udp_port
@@ -76,9 +84,11 @@ async def all_customers_status(
         else:
             entry["deployment_status"] = None
             entry["containers"] = []
-        results.append(entry)
+        return entry
 
-    return results
+    # Fetch container status for all customers concurrently instead of one
+    # blocking Docker SDK call at a time.
+    return await asyncio.gather(*[_build_entry(c) for c in customers])
 
 
 @router.get("/resources")
@@ -205,15 +215,28 @@ async def customers_local_update_status(
 
     Compares running container image IDs against locally stored images.
     No network call — safe to call on every dashboard load.
+
+    Results are cached for a few seconds since this is triggered on every
+    customer-table render (including every search keystroke) but the
+    underlying data rarely changes.
     """
+    now = time.monotonic()
+    if _update_status_cache["data"] is not None and now < _update_status_cache["expires"]:
+        return _update_status_cache["data"]
+
     config = db.query(SystemConfig).filter(SystemConfig.id == 1).first()
     if not config:
         return []
     deployments = db.query(Deployment).all()
-    results = []
-    for dep in deployments:
-        cs = image_service.get_customer_container_image_status(dep.container_prefix, config)
-        results.append({"customer_id": dep.customer_id, "needs_update": cs["needs_update"]})
+
+    async def _check(dep: Deployment) -> dict[str, Any]:
+        cs = await image_service.get_customer_container_image_status_async(dep.container_prefix, config)
+        return {"customer_id": dep.customer_id, "needs_update": cs["needs_update"]}
+
+    results = await asyncio.gather(*[_check(dep) for dep in deployments])
+    results = list(results)
+    _update_status_cache["data"] = results
+    _update_status_cache["expires"] = now + _UPDATE_STATUS_TTL_SECONDS
     return results
 
 

app/services/docker_service.py

@@ -27,13 +27,24 @@ async def _run_cmd(cmd: list[str], timeout: int = 120) -> subprocess.CompletedPr
     )
 
 
+_client: Optional[docker.DockerClient] = None
+
+
 def _get_client() -> docker.DockerClient:
-    """Return a Docker client connected via the Unix socket.
+    """Return a shared Docker client connected via the Unix socket.
+
+    The client is created once and reused — creating a new client per call
+    (as `docker.from_env()` does) re-negotiates the API version and opens a
+    fresh connection every time, which is wasteful when called once per
+    customer in a loop.
 
     Returns:
         docker.DockerClient instance.
     """
-    return docker.from_env()
+    global _client
+    if _client is None:
+        _client = docker.from_env()
+    return _client
 
 
 async def compose_up(
@@ -212,6 +223,16 @@ def get_container_status(container_prefix: str) -> list[dict[str, Any]]:
     return results
 
 
+async def get_container_status_async(container_prefix: str) -> list[dict[str, Any]]:
+    """Thread-offloaded wrapper around get_container_status().
+
+    Use this when checking status for multiple customers so the Docker SDK
+    calls run in the thread pool instead of blocking the event loop.
+    """
+    loop = asyncio.get_event_loop()
+    return await loop.run_in_executor(None, get_container_status, container_prefix)
+
+
 def get_container_logs(container_name: str, tail: int = 200) -> str:
     """Retrieve recent logs from a container.
 

app/services/image_service.py

@@ -211,6 +211,44 @@ async def pull_all_images(config) -> dict[str, Any]:
     }
 
 
+async def get_customer_container_image_status_async(container_prefix: str, config) -> dict[str, Any]:
+    """Async, thread-offloaded version of get_customer_container_image_status().
+
+    Runs the per-service `docker inspect` subprocess calls concurrently in the
+    thread pool instead of sequentially blocking the event loop — use this
+    whenever checking status for multiple customers (e.g. dashboard/search
+    badge refresh, monitoring overview).
+
+    Returns:
+        services: dict mapping service name to status info
+        needs_update: True if any service has a different image ID than locally stored
+    """
+    service_images = {
+        "management": config.netbird_management_image,
+        "signal": config.netbird_signal_image,
+        "relay": config.netbird_relay_image,
+        "dashboard": config.netbird_dashboard_image,
+    }
+    loop = asyncio.get_event_loop()
+
+    async def _check(svc: str, image: str) -> tuple[str, dict[str, Any]]:
+        container_name = f"{container_prefix}-{svc}"
+        container_id, local_id = await asyncio.gather(
+            loop.run_in_executor(None, get_container_image_id, container_name),
+            loop.run_in_executor(None, get_local_image_id, image),
+        )
+        if container_id and local_id:
+            up_to_date = container_id == local_id
+        else:
+            up_to_date = None  # container not running or image not pulled
+        return svc, {"container": container_name, "image": image, "up_to_date": up_to_date}
+
+    pairs = await asyncio.gather(*[_check(svc, image) for svc, image in service_images.items()])
+    services = dict(pairs)
+    needs_update = any(s["up_to_date"] is False for s in services.values())
+    return {"services": services, "needs_update": needs_update}
+
+
 def get_customer_container_image_status(container_prefix: str, config) -> dict[str, Any]:
     """Check which service containers are running outdated local images.
 

app/services/netbird_service.py

@@ -264,10 +264,12 @@ async def deploy_customer(db: Session, customer_id: int) -> dict[str, Any]:
             _log_action(db, customer_id, "deploy", "info",
                         "Auto-setup failed — admin must complete setup manually.")
 
-        # Step 9: Create NPM proxy host (production only)
-        npm_proxy_id = None
-        npm_stream_id = None
-        if not local_mode:
+        # Step 9: Create NPM proxy host (production only).
+        # If an existing deployment already has an NPM proxy, reuse it — this happens
+        # when keep_data=True was passed and undeploy_customer was NOT called beforehand.
+        npm_proxy_id = existing_deployment.npm_proxy_id if existing_deployment else None
+        npm_stream_id = existing_deployment.npm_stream_id if existing_deployment else None
+        if not local_mode and not npm_proxy_id:
             forward_host = npm_service._get_forward_host()
             npm_result = await npm_service.create_proxy_host(
                 api_url=config.npm_api_url,
@@ -304,9 +306,14 @@ async def deploy_customer(db: Session, customer_id: int) -> dict[str, Any]:
                     "SSL certificate not created automatically. "
                     "Please create it manually in NPM or ensure DNS resolves and port 80 is reachable, then re-deploy.",
                 )
+        elif npm_proxy_id and not local_mode:
+            _log_action(db, customer_id, "deploy", "info",
+                        f"Reusing existing NPM proxy (ID {npm_proxy_id}) — data preserved.")
 
-        # Step 10: Create Windows DNS A-record (non-fatal — failure does not abort deployment)
-        if config.dns_enabled and config.dns_server and config.dns_zone and config.dns_record_ip:
+        # Step 10: Create Windows DNS A-record (non-fatal — failure does not abort deployment).
+        # Skip if an existing deployment is being kept (DNS record already exists).
+        if config.dns_enabled and config.dns_server and config.dns_zone and config.dns_record_ip \
+                and not existing_deployment:
             try:
                 dns_result = await dns_service.create_dns_record(customer.subdomain, config)
                 if dns_result["ok"]:

app/services/npm_service.py

@@ -263,10 +263,14 @@ async def create_proxy_host(
             "location ^~ /management.ManagementService/ {\n"
             f"    grpc_pass grpc://{forward_host}:{forward_port};\n"
             "    grpc_set_header Host $host;\n"
+            "    grpc_read_timeout 3600s;\n"
+            "    grpc_send_timeout 3600s;\n"
             "}\n"
             "location ^~ /signalexchange.SignalExchange/ {\n"
             f"    grpc_pass grpc://{forward_host}:{forward_port};\n"
             "    grpc_set_header Host $host;\n"
+            "    grpc_read_timeout 3600s;\n"
+            "    grpc_send_timeout 3600s;\n"
             "}\n"
         ),
         "meta": {

static/index.html

@@ -1267,6 +1267,49 @@
         </div>
     </div>
 
+    <!-- Modal: Redeploy Confirmation -->
+    <div class="modal fade" id="redeploy-modal" tabindex="-1">
+        <div class="modal-dialog modal-lg">
+            <div class="modal-content">
+                <div class="modal-header bg-primary text-white">
+                    <h5 class="modal-title" data-i18n="redeployModal.title">Redeploy Customer</h5>
+                    <button type="button" class="btn-close btn-close-white" data-bs-dismiss="modal"></button>
+                </div>
+                <div class="modal-body">
+                    <p><span data-i18n="redeployModal.intro">How should</span> <strong id="redeploy-customer-name"></strong> <span data-i18n="redeployModal.intro2">be redeployed?</span></p>
+                    <input type="hidden" id="redeploy-customer-id">
+                    <div class="row g-3 mt-1">
+                        <div class="col-md-6">
+                            <div class="card h-100 border-success" style="cursor:pointer" onclick="confirmRedeploy(true)" id="redeploy-card-keep">
+                                <div class="card-body">
+                                    <h6 class="card-title text-success"><i class="bi bi-shield-check me-2"></i><span data-i18n="redeployModal.keepTitle">Keep Data</span></h6>
+                                    <p class="card-text small" data-i18n="redeployModal.keepDesc">Containers are stopped and restarted. The NetBird database, peer configurations, and encryption keys are preserved. Use this after a config change or image update.</p>
+                                </div>
+                                <div class="card-footer bg-success bg-opacity-10 text-success small" data-i18n="redeployModal.keepNote">
+                                    Peers stay connected after restart.
+                                </div>
+                            </div>
+                        </div>
+                        <div class="col-md-6">
+                            <div class="card h-100 border-danger" style="cursor:pointer" onclick="confirmRedeploy(false)" id="redeploy-card-fresh">
+                                <div class="card-body">
+                                    <h6 class="card-title text-danger"><i class="bi bi-trash me-2"></i><span data-i18n="redeployModal.freshTitle">Fresh Deploy</span></h6>
+                                    <p class="card-text small" data-i18n="redeployModal.freshDesc">All existing data is deleted — containers, volumes, config files, and the NetBird database. A completely new instance is created. All peers must re-enroll.</p>
+                                </div>
+                                <div class="card-footer bg-danger bg-opacity-10 text-danger small" data-i18n="redeployModal.freshNote">
+                                    All peer data is lost. Cannot be undone.
+                                </div>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+                <div class="modal-footer">
+                    <button type="button" class="btn btn-secondary" data-bs-dismiss="modal" data-i18n="common.cancel">Cancel</button>
+                </div>
+            </div>
+        </div>
+    </div>
+
     <!-- Modal: Delete Confirmation -->
     <div class="modal fade" id="delete-modal" tabindex="-1">
         <div class="modal-dialog">

static/js/app.js

@@ -667,9 +667,13 @@ async function confirmDeleteCustomer() {
 }
 
 // ---------------------------------------------------------------------------
-// Customer Actions (start/stop/restart)
+// Customer Actions (start/stop/restart/deploy)
 // ---------------------------------------------------------------------------
-async function customerAction(id, action) {
+async function customerAction(id, action, name) {
+    if (action === 'deploy') {
+        showRedeployModal(id, name);
+        return;
+    }
     try {
         await api('POST', `/customers/${id}/${action}`);
         if (currentPage === 'dashboard') loadCustomers();
@@ -679,6 +683,29 @@ async function customerAction(id, action) {
     }
 }
 
+function showRedeployModal(id, name) {
+    // Prefer passed name, fallback to dashboard table row, then ID
+    if (!name) {
+        const row = document.querySelector(`tr[data-customer-id="${id}"]`);
+        name = row ? row.querySelector('td')?.textContent?.trim() : `#${id}`;
+    }
+    document.getElementById('redeploy-customer-id').value = id;
+    document.getElementById('redeploy-customer-name').textContent = name;
+    new bootstrap.Modal(document.getElementById('redeploy-modal')).show();
+}
+
+async function confirmRedeploy(keepData) {
+    const id = document.getElementById('redeploy-customer-id').value;
+    bootstrap.Modal.getInstance(document.getElementById('redeploy-modal'))?.hide();
+    try {
+        await api('POST', `/customers/${id}/deploy?keep_data=${keepData}`);
+        if (currentPage === 'dashboard') loadCustomers();
+        if (currentCustomerId == id) viewCustomer(id);
+    } catch (err) {
+        alert(t('errors.actionFailed', { action: 'deploy', error: err.message }));
+    }
+}
+
 // ---------------------------------------------------------------------------
 // Customer Detail
 // ---------------------------------------------------------------------------
@@ -762,7 +789,7 @@ async function viewCustomer(id) {
                     <button class="btn btn-success btn-sm me-1" onclick="customerAction(${id},'start')"><i class="bi bi-play-circle me-1"></i>${t('customer.start')}</button>
                     <button class="btn btn-warning btn-sm me-1" onclick="customerAction(${id},'stop')"><i class="bi bi-stop-circle me-1"></i>${t('customer.stop')}</button>
                     <button class="btn btn-info btn-sm me-1" onclick="customerAction(${id},'restart')"><i class="bi bi-arrow-repeat me-1"></i>${t('customer.restart')}</button>
-                    <button class="btn btn-outline-primary btn-sm me-1" onclick="customerAction(${id},'deploy')"><i class="bi bi-rocket me-1"></i>${t('customer.reDeploy')}</button>
+                    <button class="btn btn-outline-primary btn-sm me-1" data-customer-name="${esc(data.name)}" onclick="customerAction(${id},'deploy',this.dataset.customerName)"><i class="bi bi-rocket me-1"></i>${t('customer.reDeploy')}</button>
                     <button class="btn btn-outline-warning btn-sm" id="btn-update-images-detail" onclick="updateCustomerImagesFromDetail(${id})">
                         <span id="update-detail-spinner" class="spinner-border spinner-border-sm d-none me-1"></span>
                         <i class="bi bi-arrow-repeat me-1"></i>${t('customer.updateImages')}

static/js/i18n.js

@@ -27,7 +27,8 @@ function detectLanguage() {
 async function loadLanguage(lang) {
     if (translations[lang]) return;
     try {
-        const resp = await fetch(`/static/lang/${lang}.json`);
+        const v = window.STATIC_VERSION ? `?v=${window.STATIC_VERSION}` : '';
+        const resp = await fetch(`/static/lang/${lang}.json${v}`);
         if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
         translations[lang] = await resp.json();
     } catch (err) {

static/lang/de.json

@@ -354,6 +354,17 @@
     "saveAndDeploy": "Speichern & Bereitstellen",
     "saveChanges": "Änderungen speichern"
   },
+  "redeployModal": {
+    "title": "Kunde neu bereitstellen",
+    "intro": "Wie soll",
+    "intro2": "neu bereitgestellt werden?",
+    "keepTitle": "Daten behalten",
+    "keepDesc": "Container werden gestoppt und neu gestartet. Die NetBird-Datenbank, Peer-Konfigurationen und Verschlüsselungsschlüssel bleiben erhalten. Verwenden Sie dies nach einer Konfigurationsänderung oder einem Image-Update.",
+    "keepNote": "Peers bleiben nach dem Neustart verbunden.",
+    "freshTitle": "Neu aufsetzen",
+    "freshDesc": "Alle bestehenden Daten werden gelöscht — Container, Volumes, Konfigurationsdateien und die NetBird-Datenbank. Eine komplett neue Instanz wird erstellt. Alle Peers müssen sich neu registrieren.",
+    "freshNote": "Alle Peer-Daten gehen verloren. Kann nicht rückgängig gemacht werden."
+  },
   "deleteModal": {
     "title": "Löschen bestätigen",
     "confirmText": "Möchten Sie den Kunden wirklich löschen:",

static/lang/en.json

@@ -107,6 +107,17 @@
     "saveAndDeploy": "Save & Deploy",
     "saveChanges": "Save Changes"
   },
+  "redeployModal": {
+    "title": "Redeploy Customer",
+    "intro": "How should",
+    "intro2": "be redeployed?",
+    "keepTitle": "Keep Data",
+    "keepDesc": "Containers are stopped and restarted. The NetBird database, peer configurations, and encryption keys are preserved. Use this after a config change or image update.",
+    "keepNote": "Peers stay connected after restart.",
+    "freshTitle": "Fresh Deploy",
+    "freshDesc": "All existing data is deleted — containers, volumes, config files, and the NetBird database. A completely new instance is created. All peers must re-enroll.",
+    "freshNote": "All peer data is lost. Cannot be undone."
+  },
   "deleteModal": {
     "title": "Confirm Deletion",
     "confirmText": "Are you sure you want to delete customer",