fix(updater): add force-recreate and logging to helper container

app/services/update_service.py

@@ -232,25 +232,111 @@ def trigger_update(config: Any, db_path: str) -> dict:
         build_env.get("GIT_BRANCH", "?"),
     )
 
-    # 5. Fire-and-forget docker compose rebuild — the container will restart itself
-    #    Use the correct project name so compose finds/replaces the right container.
-    #    Only rebuild the app service — docker-socket-proxy must not be recreated.
-    compose_cmd = [
+    # 5. Two-phase rebuild: Build image first, then swap container.
+    #    The swap will kill this process (we ARE the container), so we must
+    #    ensure the compose-up runs detached on the Docker host via a wrapper.
+    log_path = Path(BACKUP_DIR) / "update_rebuild.log"
+
+    # Phase A — build the new image (does NOT stop anything)
+    build_cmd = [
         "docker", "compose",
         "-p", "netbirdmsp-appliance",
         "-f", f"{SOURCE_DIR}/docker-compose.yml",
-        "up", "--build", "--no-deps", "-d",
+        "build", "--no-cache",
         "netbird-msp-appliance",
     ]
-    log_path = Path(BACKUP_DIR) / "update_rebuild.log"
-    log_file = open(log_path, "w")
-    subprocess.Popen(
-        compose_cmd,
-        stdout=log_file,
-        stderr=log_file,
-        env=build_env,
-    )
-    logger.info("docker compose up --build -d triggered — container will restart shortly.")
+    logger.info("Phase A: building new image …")
+    try:
+        build_result = subprocess.run(
+            build_cmd,
+            capture_output=True, text=True,
+            timeout=600,
+            env=build_env,
+        )
+        with open(log_path, "w") as f:
+            f.write(build_result.stdout)
+            f.write(build_result.stderr)
+        if build_result.returncode != 0:
+            logger.error("Image build failed: %s", build_result.stderr[:500])
+            return {
+                "ok": False,
+                "message": f"Image build failed: {build_result.stderr[:300]}",
+                "backup": backup_path,
+            }
+    except subprocess.TimeoutExpired:
+        return {"ok": False, "message": "Image build timed out after 600s.", "backup": backup_path}
+
+    logger.info("Phase A complete — image built successfully.")
+
+    # Phase B — swap the container using a helper container.
+    #    When compose recreates our container, ALL processes inside die (PID namespace
+    #    is destroyed). So we launch a *separate* helper container via 'docker run -d'
+    #    that has access to the Docker socket and runs 'docker compose up -d'.
+    #    This helper lives outside our container and survives our restart.
+
+    # Discover the host-side path of /app-source (docker volumes use host paths)
+    try:
+        inspect_result = subprocess.run(
+            ["docker", "inspect", "netbird-msp-appliance",
+             "--format", '{{range .Mounts}}{{if eq .Destination "/app-source"}}{{.Source}}{{end}}{{end}}'],
+            capture_output=True, text=True, timeout=10,
+        )
+        host_source_dir = inspect_result.stdout.strip()
+        if not host_source_dir:
+            raise ValueError("Could not find /app-source mount")
+    except Exception as exc:
+        logger.error("Failed to discover host source path: %s", exc)
+        return {"ok": False, "message": f"Could not find host source path: {exc}", "backup": backup_path}
+
+    logger.info("Host source directory: %s", host_source_dir)
+
+    env_flags = []
+    for key in ("GIT_TAG", "GIT_COMMIT", "GIT_BRANCH", "GIT_COMMIT_DATE"):
+        val = build_env.get(key, "unknown")
+        env_flags.extend(["-e", f"{key}={val}"])
+
+    # Use the same image we're already running (it has docker CLI + compose plugin)
+    own_image = "netbirdmsp-appliance-netbird-msp-appliance:latest"
+
+    helper_cmd = [
+        "docker", "run", "-d",
+        "--name", "msp-updater",
+        "-v", "/var/run/docker.sock:/var/run/docker.sock",
+        "-v", f"{host_source_dir}:{host_source_dir}:ro",
+        *env_flags,
+        own_image,
+        "sh", "-c",
+        (
+            "sleep 3 && "
+            "docker compose -p netbirdmsp-appliance "
+            f"-f {host_source_dir}/docker-compose.yml "
+            "up --force-recreate --no-deps -d netbird-msp-appliance "
+            f">> {host_source_dir}/app/backups/updater.log 2>&1"
+        ),
+    ]
+    try:
+        # Remove stale updater container if any
+        subprocess.run(
+            ["docker", "rm", "-f", "msp-updater"],
+            capture_output=True, timeout=10,
+        )
+        result = subprocess.run(
+            helper_cmd,
+            capture_output=True, text=True,
+            timeout=30,
+            env=build_env,
+        )
+        if result.returncode != 0:
+            logger.error("Failed to start updater container: %s", result.stderr.strip())
+            return {
+                "ok": False,
+                "message": f"Update-Container konnte nicht gestartet werden: {result.stderr.strip()[:200]}",
+                "backup": backup_path,
+            }
+        logger.info("Phase B: updater container started — this container will restart in ~5s.")
+    except Exception as exc:
+        logger.error("Failed to launch updater: %s", exc)
+        return {"ok": False, "message": f"Updater launch failed: {exc}", "backup": backup_path}
 
     return {
         "ok": True,

helper.txt

@@ -0,0 +1 @@
+Error response from daemon: No such container: msp-updater

logs.txt

@@ -0,0 +1,50 @@
+INFO:     127.0.0.1:35822 - "GET /api/health HTTP/1.1" 200 OK
+INFO:     127.0.0.1:33932 - "GET /api/health HTTP/1.1" 200 OK
+INFO:     127.0.0.1:50284 - "GET /api/health HTTP/1.1" 200 OK
+INFO:     172.18.0.1:49612 - "GET / HTTP/1.1" 200 OK
+INFO:     172.18.0.1:49612 - "GET /css/styles.css HTTP/1.1" 304 Not Modified
+INFO:     172.18.0.1:49610 - "GET /js/i18n.js HTTP/1.1" 304 Not Modified
+INFO:     172.18.0.1:49632 - "GET /js/app.js HTTP/1.1" 200 OK
+INFO:     172.18.0.1:49632 - "GET /lang/en.json HTTP/1.1" 200 OK
+INFO:     172.18.0.1:49632 - "GET /favicon.ico HTTP/1.1" 404 Not Found
+INFO:     172.18.0.1:49610 - "GET /lang/de.json HTTP/1.1" 200 OK
+INFO:     172.18.0.1:49610 - "GET /api/settings/branding HTTP/1.1" 200 OK
+INFO:     172.18.0.1:49610 - "GET /api/auth/azure/config HTTP/1.1" 200 OK
+INFO:     127.0.0.1:59642 - "GET /api/health HTTP/1.1" 200 OK
+2026-02-22 13:56:39,498 [WARNING] passlib.handlers.bcrypt: (trapped) error reading bcrypt version
+Traceback (most recent call last):
+  File "/usr/local/lib/python3.11/site-packages/passlib/handlers/bcrypt.py", line 620, in _load_backend_mixin
+    version = _bcrypt.__about__.__version__
+              ^^^^^^^^^^^^^^^^^
+AttributeError: module 'bcrypt' has no attribute '__about__'
+2026-02-22 13:56:39,929 [INFO] app.routers.auth: User admin logged in (provider: local).
+INFO:     172.18.0.1:36450 - "POST /api/auth/login HTTP/1.1" 200 OK
+INFO:     172.18.0.1:36462 - "GET /api/customers?page=1&per_page=25 HTTP/1.1" 200 OK
+INFO:     172.18.0.1:36450 - "GET /api/monitoring/status HTTP/1.1" 200 OK
+INFO:     127.0.0.1:54154 - "GET /api/health HTTP/1.1" 200 OK
+INFO:     172.18.0.1:54490 - "GET /api/settings/system HTTP/1.1" 200 OK
+INFO:     172.18.0.1:54490 - "GET /api/auth/mfa/status HTTP/1.1" 200 OK
+2026-02-22 13:57:10,815 [INFO] httpx: HTTP Request: GET https://git.0x26.ch/api/v1/repos/BurgerGames/NetBirdMSP-Appliance/branches/unstable "HTTP/1.1 200 OK"
+2026-02-22 13:57:10,822 [INFO] httpx: HTTP Request: GET https://git.0x26.ch/api/v1/repos/BurgerGames/NetBirdMSP-Appliance/tags?limit=1 "HTTP/1.1 200 OK"
+INFO:     172.18.0.1:57512 - "GET /api/settings/version HTTP/1.1" 200 OK
+INFO:     127.0.0.1:52478 - "GET /api/health HTTP/1.1" 200 OK
+INFO:     127.0.0.1:47310 - "GET /api/health HTTP/1.1" 200 OK
+INFO:     127.0.0.1:47530 - "GET /api/health HTTP/1.1" 200 OK
+INFO:     127.0.0.1:41918 - "GET /api/health HTTP/1.1" 200 OK
+INFO:     127.0.0.1:45108 - "GET /api/health HTTP/1.1" 200 OK
+2026-02-22 13:59:53,200 [INFO] app.services.update_service: Database backed up to /app/backups/netbird_msp_20260222_135953.db
+2026-02-22 13:59:54,630 [INFO] app.services.update_service: git pull succeeded: Already up to date.
+2026-02-22 13:59:54,740 [INFO] app.services.update_service: Rebuilding with GIT_TAG=alpha-1.4 GIT_COMMIT=ef691a4 GIT_BRANCH=unstable
+2026-02-22 13:59:54,741 [INFO] app.services.update_service: Phase A: building new image …
+2026-02-22 14:03:51,162 [INFO] app.services.update_service: Phase A complete — image built successfully.
+2026-02-22 14:03:51,242 [INFO] app.services.update_service: Host source directory: /home/sascha/NetBirdMSP-Appliance
+2026-02-22 14:03:52,032 [INFO] app.services.update_service: Phase B: updater container started — this container will restart in ~5s.
+2026-02-22 14:03:52,033 [INFO] app.routers.settings: Update triggered by admin.
+INFO:     172.18.0.1:53362 - "POST /api/settings/update HTTP/1.1" 200 OK
+INFO:     172.18.0.1:35312 - "POST /api/settings/update HTTP/1.1" 401 Unauthorized
+INFO:     127.0.0.1:35534 - "GET /api/health HTTP/1.1" 200 OK
+2026-02-22 14:04:22,366 [INFO] httpx: HTTP Request: GET https://git.0x26.ch/api/v1/repos/BurgerGames/NetBirdMSP-Appliance/branches/unstable "HTTP/1.1 200 OK"
+2026-02-22 14:04:22,376 [INFO] httpx: HTTP Request: GET https://git.0x26.ch/api/v1/repos/BurgerGames/NetBirdMSP-Appliance/tags?limit=1 "HTTP/1.1 200 OK"
+INFO:     172.18.0.1:53602 - "GET /api/settings/version HTTP/1.1" 200 OK
+INFO:     127.0.0.1:51374 - "GET /api/health HTTP/1.1" 200 OK
+INFO:     127.0.0.1:48640 - "GET /api/health HTTP/1.1" 200 OK

port.txt

@@ -0,0 +1,2 @@
+8000/tcp -> 0.0.0.0:8000
+8000/tcp -> [::]:8000