twothatIT
e53539231e
Customer container status checks looked up containers by an exact expected name. When a docker compose recreate got interrupted (e.g. a hung command previously killed the whole update-all batch on timeout), Compose could leave the old container renamed with a random hash prefix instead of removed. The exact-name lookup then found nothing, returned None, and that silently counted as "up to date" (green "Aktuell") instead of surfacing as unknown — affecting 5 customers on the appliance whose containers were actually still running under orphaned names. - _run_cmd no longer raises on subprocess timeout, so one stuck customer can't abort the rest of a batch update - repair_container_naming() self-heals orphaned hash-renamed containers by renaming them back before every status check and before recreate - update-all loop now catches per-customer exceptions instead of aborting - status responses expose "unknown" separately from "needs_update" so the UI shows a distinct grey badge instead of a false-positive green one - new settings: automatic daily update check (on/off + time), with an independent toggle for whether it also auto-recreates customer containers
406 lines
16 KiB
Python
406 lines
16 KiB
Python
"""NetBird Docker image update service.
|
|
|
|
Compares locally pulled images against Docker Hub to detect available updates.
|
|
Provides pull and per-customer container recreation functions without data loss.
|
|
"""
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import os
|
|
import subprocess
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Services that make up a customer's NetBird deployment
|
|
NETBIRD_SERVICES = ["management", "signal", "relay", "dashboard"]
|
|
|
|
|
|
class _TimeoutResult:
|
|
"""Stand-in for subprocess.CompletedProcess when a command times out.
|
|
|
|
A hung `docker compose up -d` used to raise TimeoutExpired straight out of
|
|
_run_cmd, which killed the whole update-all loop mid-recreate and left the
|
|
old container renamed-but-not-removed (orphaned with a hash-prefixed name).
|
|
Returning a failed result instead lets callers handle it gracefully and
|
|
keeps the batch loop going for the remaining customers.
|
|
"""
|
|
|
|
def __init__(self, cmd: list[str], timeout: int):
|
|
self.returncode = -1
|
|
self.stdout = ""
|
|
self.stderr = f"Command timed out after {timeout}s: {' '.join(cmd)}"
|
|
|
|
|
|
async def _run_cmd(cmd: list[str], timeout: int = 300) -> subprocess.CompletedProcess:
|
|
"""Run a subprocess command without blocking the event loop.
|
|
|
|
Never raises on timeout — returns a failed CompletedProcess-like result
|
|
instead, so a single hung docker/compose call can't abort a batch of
|
|
otherwise-independent operations (e.g. updating multiple customers).
|
|
"""
|
|
loop = asyncio.get_event_loop()
|
|
try:
|
|
return await loop.run_in_executor(
|
|
None,
|
|
lambda: subprocess.run(cmd, capture_output=True, text=True, timeout=timeout),
|
|
)
|
|
except subprocess.TimeoutExpired:
|
|
logger.error("Command timed out after %ds: %s", timeout, " ".join(cmd))
|
|
return _TimeoutResult(cmd, timeout)
|
|
|
|
|
|
def _parse_image_name(image: str) -> tuple[str, str]:
|
|
"""Split 'repo/name:tag' into ('repo/name', 'tag'). Defaults tag to 'latest'."""
|
|
if ":" in image:
|
|
name, tag = image.rsplit(":", 1)
|
|
else:
|
|
name, tag = image, "latest"
|
|
return name, tag
|
|
|
|
|
|
async def get_hub_digest(image: str) -> str | None:
|
|
"""Fetch the manifest-list digest from the Docker Registry v2 API.
|
|
|
|
Uses anonymous auth against registry-1.docker.io — does NOT pull the image.
|
|
Returns the Docker-Content-Digest header value (sha256:...) which is identical
|
|
to the digest stored in local RepoDigests after a pull, enabling correct comparison.
|
|
"""
|
|
name, tag = _parse_image_name(image)
|
|
try:
|
|
async with httpx.AsyncClient(timeout=15) as client:
|
|
# Step 1: obtain anonymous pull token
|
|
token_resp = await client.get(
|
|
"https://auth.docker.io/token",
|
|
params={"service": "registry.docker.io", "scope": f"repository:{name}:pull"},
|
|
)
|
|
if token_resp.status_code != 200:
|
|
logger.warning("Failed to get registry token for %s", image)
|
|
return None
|
|
token = token_resp.json().get("token")
|
|
|
|
# Step 2: fetch manifest — prefer manifest list (multi-arch) so the digest
|
|
# matches what `docker pull` stores in RepoDigests.
|
|
manifest_resp = await client.get(
|
|
f"https://registry-1.docker.io/v2/{name}/manifests/{tag}",
|
|
headers={
|
|
"Authorization": f"Bearer {token}",
|
|
"Accept": (
|
|
"application/vnd.docker.distribution.manifest.list.v2+json, "
|
|
"application/vnd.oci.image.index.v1+json, "
|
|
"application/vnd.docker.distribution.manifest.v2+json"
|
|
),
|
|
},
|
|
)
|
|
if manifest_resp.status_code != 200:
|
|
logger.warning("Registry API returned %d for %s", manifest_resp.status_code, image)
|
|
return None
|
|
|
|
# The Docker-Content-Digest header is the canonical digest
|
|
digest = manifest_resp.headers.get("docker-content-digest")
|
|
if digest:
|
|
return digest
|
|
return None
|
|
except Exception as exc:
|
|
logger.warning("Failed to fetch registry digest for %s: %s", image, exc)
|
|
return None
|
|
|
|
|
|
def get_local_digest(image: str) -> str | None:
|
|
"""Get the RepoDigest for a locally pulled image.
|
|
|
|
Returns the digest (sha256:...) or None if image not found locally.
|
|
"""
|
|
try:
|
|
result = subprocess.run(
|
|
["docker", "image", "inspect", image, "--format", "{{json .RepoDigests}}"],
|
|
capture_output=True, text=True, timeout=10,
|
|
)
|
|
if result.returncode != 0:
|
|
return None
|
|
digests = json.loads(result.stdout.strip())
|
|
if not digests:
|
|
return None
|
|
# RepoDigests look like "netbirdio/management@sha256:abc..."
|
|
for d in digests:
|
|
if "@" in d:
|
|
return d.split("@", 1)[1]
|
|
return None
|
|
except Exception as exc:
|
|
logger.warning("Failed to inspect local image %s: %s", image, exc)
|
|
return None
|
|
|
|
|
|
def get_container_image_id(container_name: str) -> str | None:
|
|
"""Get the full image ID (sha256:...) of a running or stopped container."""
|
|
try:
|
|
result = subprocess.run(
|
|
["docker", "inspect", container_name, "--format", "{{.Image}}"],
|
|
capture_output=True, text=True, timeout=10,
|
|
)
|
|
if result.returncode != 0:
|
|
return None
|
|
return result.stdout.strip() or None
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def repair_container_naming(container_prefix: str, services: list[str] = NETBIRD_SERVICES) -> list[str]:
|
|
"""Rename orphaned containers back to their expected compose name.
|
|
|
|
When a `docker compose up -d` is interrupted mid-recreate (e.g. a timeout
|
|
killing the process), Compose can leave the *old* container renamed with a
|
|
random hash prefix (e.g. "4e45e71fcb7b_netbird-acme-management") instead
|
|
of removing it, while never creating the correctly-named replacement. The
|
|
container itself keeps running fine — it's just invisible to every lookup
|
|
that expects the exact name, which used to silently read as "no container
|
|
found" and get reported as "up to date" instead of "unknown".
|
|
|
|
This finds any such orphan (a container whose name *contains* the expected
|
|
name but isn't an exact match) and, only when no container already holds
|
|
the exact expected name, renames it back. Safe no-op otherwise.
|
|
|
|
Returns the list of service names that were repaired.
|
|
"""
|
|
repaired = []
|
|
for svc in services:
|
|
expected_name = f"{container_prefix}-{svc}"
|
|
exact = subprocess.run(
|
|
["docker", "inspect", expected_name, "--format", "{{.Id}}"],
|
|
capture_output=True, text=True, timeout=10,
|
|
)
|
|
if exact.returncode == 0:
|
|
continue # already correctly named
|
|
|
|
found = subprocess.run(
|
|
["docker", "ps", "-a", "--filter", f"name={expected_name}", "--format", "{{.Names}}"],
|
|
capture_output=True, text=True, timeout=10,
|
|
)
|
|
candidates = [n for n in found.stdout.strip().splitlines() if n and n != expected_name]
|
|
if not candidates:
|
|
continue # container genuinely doesn't exist (not deployed / not running)
|
|
|
|
orphan = candidates[0]
|
|
rename = subprocess.run(
|
|
["docker", "rename", orphan, expected_name],
|
|
capture_output=True, text=True, timeout=10,
|
|
)
|
|
if rename.returncode == 0:
|
|
logger.warning(
|
|
"Repaired orphaned container naming for %s: '%s' -> '%s'",
|
|
container_prefix, orphan, expected_name,
|
|
)
|
|
repaired.append(svc)
|
|
else:
|
|
logger.error(
|
|
"Failed to repair orphaned container '%s' -> '%s': %s",
|
|
orphan, expected_name, rename.stderr,
|
|
)
|
|
return repaired
|
|
|
|
|
|
def get_local_image_id(image: str) -> str | None:
|
|
"""Get the full image ID (sha256:...) of a locally stored image."""
|
|
try:
|
|
result = subprocess.run(
|
|
["docker", "image", "inspect", image, "--format", "{{.Id}}"],
|
|
capture_output=True, text=True, timeout=10,
|
|
)
|
|
if result.returncode != 0:
|
|
return None
|
|
return result.stdout.strip() or None
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
async def check_image_status(image: str) -> dict[str, Any]:
|
|
"""Check whether a configured image has an update available on Docker Hub.
|
|
|
|
Returns a dict with:
|
|
image: the image name:tag
|
|
local_digest: digest of locally cached image (or None)
|
|
hub_digest: latest digest from Docker Hub (or None)
|
|
update_available: True if hub_digest differs from local_digest
|
|
"""
|
|
hub_digest, local_digest = await asyncio.gather(
|
|
get_hub_digest(image),
|
|
asyncio.get_event_loop().run_in_executor(None, get_local_digest, image),
|
|
)
|
|
|
|
if hub_digest and local_digest:
|
|
update_available = hub_digest != local_digest
|
|
elif hub_digest and not local_digest:
|
|
# Image not pulled locally yet — needs pull
|
|
update_available = True
|
|
else:
|
|
update_available = False
|
|
|
|
return {
|
|
"image": image,
|
|
"local_digest": local_digest,
|
|
"hub_digest": hub_digest,
|
|
"update_available": update_available,
|
|
}
|
|
|
|
|
|
async def check_all_images(config) -> dict[str, Any]:
|
|
"""Check all 4 configured NetBird images for available updates.
|
|
|
|
Returns a dict with:
|
|
images: dict mapping image name -> status dict
|
|
any_update_available: bool
|
|
"""
|
|
images = [
|
|
config.netbird_management_image,
|
|
config.netbird_signal_image,
|
|
config.netbird_relay_image,
|
|
config.netbird_dashboard_image,
|
|
]
|
|
results = await asyncio.gather(*[check_image_status(img) for img in images])
|
|
by_image = {r["image"]: r for r in results}
|
|
any_update = any(r["update_available"] for r in results)
|
|
return {"images": by_image, "any_update_available": any_update}
|
|
|
|
|
|
async def pull_image(image: str) -> dict[str, Any]:
|
|
"""Pull a Docker image. Returns success/error dict."""
|
|
logger.info("Pulling image: %s", image)
|
|
result = await _run_cmd(["docker", "pull", image], timeout=600)
|
|
if result.returncode != 0:
|
|
logger.error("Failed to pull %s: %s", image, result.stderr)
|
|
return {"image": image, "success": False, "error": result.stderr[:500]}
|
|
return {"image": image, "success": True}
|
|
|
|
|
|
async def pull_all_images(config) -> dict[str, Any]:
|
|
"""Pull all 4 configured NetBird images. Returns results per image."""
|
|
images = [
|
|
config.netbird_management_image,
|
|
config.netbird_signal_image,
|
|
config.netbird_relay_image,
|
|
config.netbird_dashboard_image,
|
|
]
|
|
results = await asyncio.gather(*[pull_image(img) for img in images])
|
|
return {
|
|
"results": {r["image"]: r for r in results},
|
|
"all_success": all(r["success"] for r in results),
|
|
}
|
|
|
|
|
|
async def get_customer_container_image_status_async(container_prefix: str, config) -> dict[str, Any]:
|
|
"""Async, thread-offloaded version of get_customer_container_image_status().
|
|
|
|
Runs the per-service `docker inspect` subprocess calls concurrently in the
|
|
thread pool instead of sequentially blocking the event loop — use this
|
|
whenever checking status for multiple customers (e.g. dashboard/search
|
|
badge refresh, monitoring overview).
|
|
|
|
Returns:
|
|
services: dict mapping service name to status info
|
|
needs_update: True if any service has a different image ID than locally stored
|
|
"""
|
|
service_images = {
|
|
"management": config.netbird_management_image,
|
|
"signal": config.netbird_signal_image,
|
|
"relay": config.netbird_relay_image,
|
|
"dashboard": config.netbird_dashboard_image,
|
|
}
|
|
loop = asyncio.get_event_loop()
|
|
|
|
# Self-heal any container left orphaned under a hash-prefixed name by a
|
|
# previously interrupted recreate, so the lookups below find it by its
|
|
# real, expected name instead of silently returning "not found".
|
|
await loop.run_in_executor(
|
|
None, repair_container_naming, container_prefix, list(service_images.keys())
|
|
)
|
|
|
|
async def _check(svc: str, image: str) -> tuple[str, dict[str, Any]]:
|
|
container_name = f"{container_prefix}-{svc}"
|
|
container_id, local_id = await asyncio.gather(
|
|
loop.run_in_executor(None, get_container_image_id, container_name),
|
|
loop.run_in_executor(None, get_local_image_id, image),
|
|
)
|
|
if container_id and local_id:
|
|
up_to_date = container_id == local_id
|
|
else:
|
|
up_to_date = None # container not running or image not pulled
|
|
return svc, {"container": container_name, "image": image, "up_to_date": up_to_date}
|
|
|
|
pairs = await asyncio.gather(*[_check(svc, image) for svc, image in service_images.items()])
|
|
services = dict(pairs)
|
|
needs_update = any(s["up_to_date"] is False for s in services.values())
|
|
unknown = any(s["up_to_date"] is None for s in services.values())
|
|
return {"services": services, "needs_update": needs_update, "unknown": unknown}
|
|
|
|
|
|
def get_customer_container_image_status(container_prefix: str, config) -> dict[str, Any]:
|
|
"""Check which service containers are running outdated local images.
|
|
|
|
Compares each running container's image ID against the locally stored image ID
|
|
for the configured image tag. This is a local check — no network call.
|
|
|
|
Returns:
|
|
services: dict mapping service name to status info
|
|
needs_update: True if any service has a different image ID than locally stored
|
|
"""
|
|
service_images = {
|
|
"management": config.netbird_management_image,
|
|
"signal": config.netbird_signal_image,
|
|
"relay": config.netbird_relay_image,
|
|
"dashboard": config.netbird_dashboard_image,
|
|
}
|
|
|
|
# Self-heal any container left orphaned under a hash-prefixed name by a
|
|
# previously interrupted recreate (see repair_container_naming docstring).
|
|
repair_container_naming(container_prefix, list(service_images.keys()))
|
|
|
|
services: dict[str, Any] = {}
|
|
for svc, image in service_images.items():
|
|
container_name = f"{container_prefix}-{svc}"
|
|
container_id = get_container_image_id(container_name)
|
|
local_id = get_local_image_id(image)
|
|
if container_id and local_id:
|
|
up_to_date = container_id == local_id
|
|
else:
|
|
up_to_date = None # container not running or image not pulled
|
|
services[svc] = {
|
|
"container": container_name,
|
|
"image": image,
|
|
"up_to_date": up_to_date,
|
|
}
|
|
needs_update = any(s["up_to_date"] is False for s in services.values())
|
|
unknown = any(s["up_to_date"] is None for s in services.values())
|
|
return {"services": services, "needs_update": needs_update, "unknown": unknown}
|
|
|
|
|
|
async def update_customer_containers(instance_dir: str, project_name: str) -> dict[str, Any]:
|
|
"""Recreate customer containers to pick up newly pulled images.
|
|
|
|
Runs `docker compose up -d` in the customer's instance directory.
|
|
Images must already be pulled. Bind-mounted data is preserved — no data loss.
|
|
"""
|
|
compose_file = os.path.join(instance_dir, "docker-compose.yml")
|
|
if not os.path.isfile(compose_file):
|
|
return {"success": False, "error": f"docker-compose.yml not found at {compose_file}"}
|
|
|
|
# Repair any container still orphaned under a hash-prefixed name from a
|
|
# previous interrupted recreate before Compose tries to touch it again —
|
|
# otherwise Compose keeps colliding with the same stuck rename.
|
|
loop = asyncio.get_event_loop()
|
|
await loop.run_in_executor(None, repair_container_naming, project_name)
|
|
|
|
cmd = [
|
|
"docker", "compose",
|
|
"-f", compose_file,
|
|
"-p", project_name,
|
|
"up", "-d", "--remove-orphans",
|
|
]
|
|
logger.info("Updating containers for %s", project_name)
|
|
result = await _run_cmd(cmd, timeout=300)
|
|
if result.returncode != 0:
|
|
return {"success": False, "error": result.stderr[:1000]}
|
|
return {"success": True}
|