"""NetBird Docker image update service. Compares locally pulled images against Docker Hub to detect available updates. Provides pull and per-customer container recreation functions without data loss. """ import asyncio import json import logging import os import subprocess from typing import Any import httpx logger = logging.getLogger(__name__) # Services that make up a customer's NetBird deployment NETBIRD_SERVICES = ["management", "signal", "relay", "dashboard"] class _TimeoutResult: """Stand-in for subprocess.CompletedProcess when a command times out. A hung `docker compose up -d` used to raise TimeoutExpired straight out of _run_cmd, which killed the whole update-all loop mid-recreate and left the old container renamed-but-not-removed (orphaned with a hash-prefixed name). Returning a failed result instead lets callers handle it gracefully and keeps the batch loop going for the remaining customers. """ def __init__(self, cmd: list[str], timeout: int): self.returncode = -1 self.stdout = "" self.stderr = f"Command timed out after {timeout}s: {' '.join(cmd)}" async def _run_cmd(cmd: list[str], timeout: int = 300) -> subprocess.CompletedProcess: """Run a subprocess command without blocking the event loop. Never raises on timeout — returns a failed CompletedProcess-like result instead, so a single hung docker/compose call can't abort a batch of otherwise-independent operations (e.g. updating multiple customers). """ loop = asyncio.get_event_loop() try: return await loop.run_in_executor( None, lambda: subprocess.run(cmd, capture_output=True, text=True, timeout=timeout), ) except subprocess.TimeoutExpired: logger.error("Command timed out after %ds: %s", timeout, " ".join(cmd)) return _TimeoutResult(cmd, timeout) def _parse_image_name(image: str) -> tuple[str, str]: """Split 'repo/name:tag' into ('repo/name', 'tag'). Defaults tag to 'latest'.""" if ":" in image: name, tag = image.rsplit(":", 1) else: name, tag = image, "latest" return name, tag async def get_hub_digest(image: str) -> str | None: """Fetch the manifest-list digest from the Docker Registry v2 API. Uses anonymous auth against registry-1.docker.io — does NOT pull the image. Returns the Docker-Content-Digest header value (sha256:...) which is identical to the digest stored in local RepoDigests after a pull, enabling correct comparison. """ name, tag = _parse_image_name(image) try: async with httpx.AsyncClient(timeout=15) as client: # Step 1: obtain anonymous pull token token_resp = await client.get( "https://auth.docker.io/token", params={"service": "registry.docker.io", "scope": f"repository:{name}:pull"}, ) if token_resp.status_code != 200: logger.warning("Failed to get registry token for %s", image) return None token = token_resp.json().get("token") # Step 2: fetch manifest — prefer manifest list (multi-arch) so the digest # matches what `docker pull` stores in RepoDigests. manifest_resp = await client.get( f"https://registry-1.docker.io/v2/{name}/manifests/{tag}", headers={ "Authorization": f"Bearer {token}", "Accept": ( "application/vnd.docker.distribution.manifest.list.v2+json, " "application/vnd.oci.image.index.v1+json, " "application/vnd.docker.distribution.manifest.v2+json" ), }, ) if manifest_resp.status_code != 200: logger.warning("Registry API returned %d for %s", manifest_resp.status_code, image) return None # The Docker-Content-Digest header is the canonical digest digest = manifest_resp.headers.get("docker-content-digest") if digest: return digest return None except Exception as exc: logger.warning("Failed to fetch registry digest for %s: %s", image, exc) return None def get_local_digest(image: str) -> str | None: """Get the RepoDigest for a locally pulled image. Returns the digest (sha256:...) or None if image not found locally. """ try: result = subprocess.run( ["docker", "image", "inspect", image, "--format", "{{json .RepoDigests}}"], capture_output=True, text=True, timeout=10, ) if result.returncode != 0: return None digests = json.loads(result.stdout.strip()) if not digests: return None # RepoDigests look like "netbirdio/management@sha256:abc..." for d in digests: if "@" in d: return d.split("@", 1)[1] return None except Exception as exc: logger.warning("Failed to inspect local image %s: %s", image, exc) return None def get_container_image_id(container_name: str) -> str | None: """Get the full image ID (sha256:...) of a running or stopped container.""" try: result = subprocess.run( ["docker", "inspect", container_name, "--format", "{{.Image}}"], capture_output=True, text=True, timeout=10, ) if result.returncode != 0: return None return result.stdout.strip() or None except Exception: return None def repair_container_naming(container_prefix: str, services: list[str] = NETBIRD_SERVICES) -> list[str]: """Rename orphaned containers back to their expected compose name. When a `docker compose up -d` is interrupted mid-recreate (e.g. a timeout killing the process), Compose can leave the *old* container renamed with a random hash prefix (e.g. "4e45e71fcb7b_netbird-acme-management") instead of removing it, while never creating the correctly-named replacement. The container itself keeps running fine — it's just invisible to every lookup that expects the exact name, which used to silently read as "no container found" and get reported as "up to date" instead of "unknown". This finds any such orphan (a container whose name *contains* the expected name but isn't an exact match) and, only when no container already holds the exact expected name, renames it back. Safe no-op otherwise. Returns the list of service names that were repaired. """ repaired = [] for svc in services: expected_name = f"{container_prefix}-{svc}" exact = subprocess.run( ["docker", "inspect", expected_name, "--format", "{{.Id}}"], capture_output=True, text=True, timeout=10, ) if exact.returncode == 0: continue # already correctly named found = subprocess.run( ["docker", "ps", "-a", "--filter", f"name={expected_name}", "--format", "{{.Names}}"], capture_output=True, text=True, timeout=10, ) candidates = [n for n in found.stdout.strip().splitlines() if n and n != expected_name] if not candidates: continue # container genuinely doesn't exist (not deployed / not running) orphan = candidates[0] rename = subprocess.run( ["docker", "rename", orphan, expected_name], capture_output=True, text=True, timeout=10, ) if rename.returncode == 0: logger.warning( "Repaired orphaned container naming for %s: '%s' -> '%s'", container_prefix, orphan, expected_name, ) repaired.append(svc) else: logger.error( "Failed to repair orphaned container '%s' -> '%s': %s", orphan, expected_name, rename.stderr, ) return repaired def get_local_image_id(image: str) -> str | None: """Get the full image ID (sha256:...) of a locally stored image.""" try: result = subprocess.run( ["docker", "image", "inspect", image, "--format", "{{.Id}}"], capture_output=True, text=True, timeout=10, ) if result.returncode != 0: return None return result.stdout.strip() or None except Exception: return None async def check_image_status(image: str) -> dict[str, Any]: """Check whether a configured image has an update available on Docker Hub. Returns a dict with: image: the image name:tag local_digest: digest of locally cached image (or None) hub_digest: latest digest from Docker Hub (or None) update_available: True if hub_digest differs from local_digest """ hub_digest, local_digest = await asyncio.gather( get_hub_digest(image), asyncio.get_event_loop().run_in_executor(None, get_local_digest, image), ) if hub_digest and local_digest: update_available = hub_digest != local_digest elif hub_digest and not local_digest: # Image not pulled locally yet — needs pull update_available = True else: update_available = False return { "image": image, "local_digest": local_digest, "hub_digest": hub_digest, "update_available": update_available, } async def check_all_images(config) -> dict[str, Any]: """Check all 4 configured NetBird images for available updates. Returns a dict with: images: dict mapping image name -> status dict any_update_available: bool """ images = [ config.netbird_management_image, config.netbird_signal_image, config.netbird_relay_image, config.netbird_dashboard_image, ] results = await asyncio.gather(*[check_image_status(img) for img in images]) by_image = {r["image"]: r for r in results} any_update = any(r["update_available"] for r in results) return {"images": by_image, "any_update_available": any_update} async def pull_image(image: str) -> dict[str, Any]: """Pull a Docker image. Returns success/error dict.""" logger.info("Pulling image: %s", image) result = await _run_cmd(["docker", "pull", image], timeout=600) if result.returncode != 0: logger.error("Failed to pull %s: %s", image, result.stderr) return {"image": image, "success": False, "error": result.stderr[:500]} return {"image": image, "success": True} async def pull_all_images(config) -> dict[str, Any]: """Pull all 4 configured NetBird images. Returns results per image.""" images = [ config.netbird_management_image, config.netbird_signal_image, config.netbird_relay_image, config.netbird_dashboard_image, ] results = await asyncio.gather(*[pull_image(img) for img in images]) return { "results": {r["image"]: r for r in results}, "all_success": all(r["success"] for r in results), } async def get_customer_container_image_status_async(container_prefix: str, config) -> dict[str, Any]: """Async, thread-offloaded version of get_customer_container_image_status(). Runs the per-service `docker inspect` subprocess calls concurrently in the thread pool instead of sequentially blocking the event loop — use this whenever checking status for multiple customers (e.g. dashboard/search badge refresh, monitoring overview). Returns: services: dict mapping service name to status info needs_update: True if any service has a different image ID than locally stored """ service_images = { "management": config.netbird_management_image, "signal": config.netbird_signal_image, "relay": config.netbird_relay_image, "dashboard": config.netbird_dashboard_image, } loop = asyncio.get_event_loop() # Self-heal any container left orphaned under a hash-prefixed name by a # previously interrupted recreate, so the lookups below find it by its # real, expected name instead of silently returning "not found". await loop.run_in_executor( None, repair_container_naming, container_prefix, list(service_images.keys()) ) async def _check(svc: str, image: str) -> tuple[str, dict[str, Any]]: container_name = f"{container_prefix}-{svc}" container_id, local_id = await asyncio.gather( loop.run_in_executor(None, get_container_image_id, container_name), loop.run_in_executor(None, get_local_image_id, image), ) if container_id and local_id: up_to_date = container_id == local_id else: up_to_date = None # container not running or image not pulled return svc, {"container": container_name, "image": image, "up_to_date": up_to_date} pairs = await asyncio.gather(*[_check(svc, image) for svc, image in service_images.items()]) services = dict(pairs) needs_update = any(s["up_to_date"] is False for s in services.values()) unknown = any(s["up_to_date"] is None for s in services.values()) return {"services": services, "needs_update": needs_update, "unknown": unknown} def get_customer_container_image_status(container_prefix: str, config) -> dict[str, Any]: """Check which service containers are running outdated local images. Compares each running container's image ID against the locally stored image ID for the configured image tag. This is a local check — no network call. Returns: services: dict mapping service name to status info needs_update: True if any service has a different image ID than locally stored """ service_images = { "management": config.netbird_management_image, "signal": config.netbird_signal_image, "relay": config.netbird_relay_image, "dashboard": config.netbird_dashboard_image, } # Self-heal any container left orphaned under a hash-prefixed name by a # previously interrupted recreate (see repair_container_naming docstring). repair_container_naming(container_prefix, list(service_images.keys())) services: dict[str, Any] = {} for svc, image in service_images.items(): container_name = f"{container_prefix}-{svc}" container_id = get_container_image_id(container_name) local_id = get_local_image_id(image) if container_id and local_id: up_to_date = container_id == local_id else: up_to_date = None # container not running or image not pulled services[svc] = { "container": container_name, "image": image, "up_to_date": up_to_date, } needs_update = any(s["up_to_date"] is False for s in services.values()) unknown = any(s["up_to_date"] is None for s in services.values()) return {"services": services, "needs_update": needs_update, "unknown": unknown} async def update_customer_containers(instance_dir: str, project_name: str) -> dict[str, Any]: """Recreate customer containers to pick up newly pulled images. Runs `docker compose up -d` in the customer's instance directory. Images must already be pulled. Bind-mounted data is preserved — no data loss. """ compose_file = os.path.join(instance_dir, "docker-compose.yml") if not os.path.isfile(compose_file): return {"success": False, "error": f"docker-compose.yml not found at {compose_file}"} # Repair any container still orphaned under a hash-prefixed name from a # previous interrupted recreate before Compose tries to touch it again — # otherwise Compose keeps colliding with the same stuck rename. loop = asyncio.get_event_loop() await loop.run_in_executor(None, repair_container_naming, project_name) cmd = [ "docker", "compose", "-f", compose_file, "-p", project_name, "up", "-d", "--remove-orphans", ] logger.info("Updating containers for %s", project_name) result = await _run_cmd(cmd, timeout=300) if result.returncode != 0: return {"success": False, "error": result.stderr[:1000]} return {"success": True}