diff --git a/app/database.py b/app/database.py
index 967ac6b..c32f965 100644
--- a/app/database.py
+++ b/app/database.py
@@ -122,6 +122,11 @@ def _run_migrations() -> None:
("system_config", "git_repo_url", "TEXT"),
("system_config", "git_branch", "TEXT DEFAULT 'main'"),
("system_config", "git_token_encrypted", "TEXT"),
+ # Automatic NetBird image update check/apply
+ ("system_config", "auto_update_check_enabled", "BOOLEAN DEFAULT 0"),
+ ("system_config", "auto_update_check_time", "TEXT DEFAULT '03:00'"),
+ ("system_config", "auto_update_apply_enabled", "BOOLEAN DEFAULT 0"),
+ ("system_config", "auto_update_last_run_at", "TEXT"),
]
for table, column, col_type in migrations:
if not _has_column(table, column):
diff --git a/app/main.py b/app/main.py
index a14f58d..d0b9e2e 100644
--- a/app/main.py
+++ b/app/main.py
@@ -13,6 +13,7 @@ from slowapi.errors import RateLimitExceeded
from app.database import init_db
from app.limiter import limiter
from app.routers import auth, customers, deployments, monitoring, settings, users
+from app.services import scheduler_service
# ---------------------------------------------------------------------------
# Logging
@@ -140,3 +141,10 @@ async def startup_event():
logger.info("Starting NetBird MSP Appliance...")
init_db()
logger.info("Database initialized.")
+ scheduler_service.start()
+
+
+@app.on_event("shutdown")
+async def shutdown_event():
+ """Stop background tasks on shutdown."""
+ scheduler_service.stop()
diff --git a/app/models.py b/app/models.py
index 20c3303..02fd990 100644
--- a/app/models.py
+++ b/app/models.py
@@ -199,6 +199,12 @@ class SystemConfig(Base):
git_branch: Mapped[Optional[str]] = mapped_column(String(100), default="main")
git_token_encrypted: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+ # Automatic NetBird image update check/apply
+ auto_update_check_enabled: Mapped[bool] = mapped_column(Boolean, default=False)
+ auto_update_check_time: Mapped[Optional[str]] = mapped_column(String(5), default="03:00")
+ auto_update_apply_enabled: Mapped[bool] = mapped_column(Boolean, default=False)
+ auto_update_last_run_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
+
created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
updated_at: Mapped[datetime] = mapped_column(
DateTime, default=datetime.utcnow, onupdate=datetime.utcnow
@@ -253,6 +259,12 @@ class SystemConfig(Base):
"git_repo_url": self.git_repo_url or "",
"git_branch": self.git_branch or "main",
"git_token_set": bool(self.git_token_encrypted),
+ "auto_update_check_enabled": bool(self.auto_update_check_enabled),
+ "auto_update_check_time": self.auto_update_check_time or "03:00",
+ "auto_update_apply_enabled": bool(self.auto_update_apply_enabled),
+ "auto_update_last_run_at": (
+ self.auto_update_last_run_at.isoformat() if self.auto_update_last_run_at else None
+ ),
"created_at": self.created_at.isoformat() if self.created_at else None,
"updated_at": self.updated_at.isoformat() if self.updated_at else None,
}
diff --git a/app/routers/monitoring.py b/app/routers/monitoring.py
index cb587bb..1f81f61 100644
--- a/app/routers/monitoring.py
+++ b/app/routers/monitoring.py
@@ -159,6 +159,7 @@ async def check_image_updates(
"subdomain": customer.subdomain,
"container_prefix": dep.container_prefix,
"needs_update": cs["needs_update"],
+ "unknown": cs.get("unknown", False),
"services": cs["services"],
})
@@ -231,7 +232,7 @@ async def customers_local_update_status(
async def _check(dep: Deployment) -> dict[str, Any]:
cs = await image_service.get_customer_container_image_status_async(dep.container_prefix, config)
- return {"customer_id": dep.customer_id, "needs_update": cs["needs_update"]}
+ return {"customer_id": dep.customer_id, "needs_update": cs["needs_update"], "unknown": cs.get("unknown", False)}
results = await asyncio.gather(*[_check(dep) for dep in deployments])
results = list(results)
@@ -274,19 +275,27 @@ async def update_all_customers(
if not to_update:
return {"message": "All customers are already up to date.", "updated": 0, "results": []}
- # Update customers sequentially — one at a time
+ # Update customers sequentially — one at a time. A failure for one
+ # customer (e.g. a hung docker compose call) must not abort the rest of
+ # the batch, otherwise later customers silently never get updated.
update_results = []
for entry in to_update:
- res = await image_service.update_customer_containers(
- entry["instance_dir"], entry["project_name"]
- )
- ok = res["success"]
- logger.info("Updated %s: %s", entry["project_name"], "OK" if ok else res.get("error"))
+ try:
+ res = await image_service.update_customer_containers(
+ entry["instance_dir"], entry["project_name"]
+ )
+ ok = res["success"]
+ error = res.get("error")
+ except Exception as exc:
+ logger.exception("Unexpected error updating %s", entry["project_name"])
+ ok = False
+ error = str(exc)
+ logger.info("Updated %s: %s", entry["project_name"], "OK" if ok else error)
update_results.append({
"customer_name": entry["customer_name"],
"customer_id": entry["customer_id"],
"success": ok,
- "error": res.get("error"),
+ "error": error,
})
success_count = sum(1 for r in update_results if r["success"])
diff --git a/app/services/image_service.py b/app/services/image_service.py
index fbbf443..2e841ba 100644
--- a/app/services/image_service.py
+++ b/app/services/image_service.py
@@ -19,13 +19,38 @@ logger = logging.getLogger(__name__)
NETBIRD_SERVICES = ["management", "signal", "relay", "dashboard"]
+class _TimeoutResult:
+ """Stand-in for subprocess.CompletedProcess when a command times out.
+
+ A hung `docker compose up -d` used to raise TimeoutExpired straight out of
+ _run_cmd, which killed the whole update-all loop mid-recreate and left the
+ old container renamed-but-not-removed (orphaned with a hash-prefixed name).
+ Returning a failed result instead lets callers handle it gracefully and
+ keeps the batch loop going for the remaining customers.
+ """
+
+ def __init__(self, cmd: list[str], timeout: int):
+ self.returncode = -1
+ self.stdout = ""
+ self.stderr = f"Command timed out after {timeout}s: {' '.join(cmd)}"
+
+
async def _run_cmd(cmd: list[str], timeout: int = 300) -> subprocess.CompletedProcess:
- """Run a subprocess command without blocking the event loop."""
+ """Run a subprocess command without blocking the event loop.
+
+ Never raises on timeout — returns a failed CompletedProcess-like result
+ instead, so a single hung docker/compose call can't abort a batch of
+ otherwise-independent operations (e.g. updating multiple customers).
+ """
loop = asyncio.get_event_loop()
- return await loop.run_in_executor(
- None,
- lambda: subprocess.run(cmd, capture_output=True, text=True, timeout=timeout),
- )
+ try:
+ return await loop.run_in_executor(
+ None,
+ lambda: subprocess.run(cmd, capture_output=True, text=True, timeout=timeout),
+ )
+ except subprocess.TimeoutExpired:
+ logger.error("Command timed out after %ds: %s", timeout, " ".join(cmd))
+ return _TimeoutResult(cmd, timeout)
def _parse_image_name(image: str) -> tuple[str, str]:
@@ -123,6 +148,60 @@ def get_container_image_id(container_name: str) -> str | None:
return None
+def repair_container_naming(container_prefix: str, services: list[str] = NETBIRD_SERVICES) -> list[str]:
+ """Rename orphaned containers back to their expected compose name.
+
+ When a `docker compose up -d` is interrupted mid-recreate (e.g. a timeout
+ killing the process), Compose can leave the *old* container renamed with a
+ random hash prefix (e.g. "4e45e71fcb7b_netbird-acme-management") instead
+ of removing it, while never creating the correctly-named replacement. The
+ container itself keeps running fine — it's just invisible to every lookup
+ that expects the exact name, which used to silently read as "no container
+ found" and get reported as "up to date" instead of "unknown".
+
+ This finds any such orphan (a container whose name *contains* the expected
+ name but isn't an exact match) and, only when no container already holds
+ the exact expected name, renames it back. Safe no-op otherwise.
+
+ Returns the list of service names that were repaired.
+ """
+ repaired = []
+ for svc in services:
+ expected_name = f"{container_prefix}-{svc}"
+ exact = subprocess.run(
+ ["docker", "inspect", expected_name, "--format", "{{.Id}}"],
+ capture_output=True, text=True, timeout=10,
+ )
+ if exact.returncode == 0:
+ continue # already correctly named
+
+ found = subprocess.run(
+ ["docker", "ps", "-a", "--filter", f"name={expected_name}", "--format", "{{.Names}}"],
+ capture_output=True, text=True, timeout=10,
+ )
+ candidates = [n for n in found.stdout.strip().splitlines() if n and n != expected_name]
+ if not candidates:
+ continue # container genuinely doesn't exist (not deployed / not running)
+
+ orphan = candidates[0]
+ rename = subprocess.run(
+ ["docker", "rename", orphan, expected_name],
+ capture_output=True, text=True, timeout=10,
+ )
+ if rename.returncode == 0:
+ logger.warning(
+ "Repaired orphaned container naming for %s: '%s' -> '%s'",
+ container_prefix, orphan, expected_name,
+ )
+ repaired.append(svc)
+ else:
+ logger.error(
+ "Failed to repair orphaned container '%s' -> '%s': %s",
+ orphan, expected_name, rename.stderr,
+ )
+ return repaired
+
+
def get_local_image_id(image: str) -> str | None:
"""Get the full image ID (sha256:...) of a locally stored image."""
try:
@@ -231,6 +310,13 @@ async def get_customer_container_image_status_async(container_prefix: str, confi
}
loop = asyncio.get_event_loop()
+ # Self-heal any container left orphaned under a hash-prefixed name by a
+ # previously interrupted recreate, so the lookups below find it by its
+ # real, expected name instead of silently returning "not found".
+ await loop.run_in_executor(
+ None, repair_container_naming, container_prefix, list(service_images.keys())
+ )
+
async def _check(svc: str, image: str) -> tuple[str, dict[str, Any]]:
container_name = f"{container_prefix}-{svc}"
container_id, local_id = await asyncio.gather(
@@ -246,7 +332,8 @@ async def get_customer_container_image_status_async(container_prefix: str, confi
pairs = await asyncio.gather(*[_check(svc, image) for svc, image in service_images.items()])
services = dict(pairs)
needs_update = any(s["up_to_date"] is False for s in services.values())
- return {"services": services, "needs_update": needs_update}
+ unknown = any(s["up_to_date"] is None for s in services.values())
+ return {"services": services, "needs_update": needs_update, "unknown": unknown}
def get_customer_container_image_status(container_prefix: str, config) -> dict[str, Any]:
@@ -265,6 +352,11 @@ def get_customer_container_image_status(container_prefix: str, config) -> dict[s
"relay": config.netbird_relay_image,
"dashboard": config.netbird_dashboard_image,
}
+
+ # Self-heal any container left orphaned under a hash-prefixed name by a
+ # previously interrupted recreate (see repair_container_naming docstring).
+ repair_container_naming(container_prefix, list(service_images.keys()))
+
services: dict[str, Any] = {}
for svc, image in service_images.items():
container_name = f"{container_prefix}-{svc}"
@@ -280,7 +372,8 @@ def get_customer_container_image_status(container_prefix: str, config) -> dict[s
"up_to_date": up_to_date,
}
needs_update = any(s["up_to_date"] is False for s in services.values())
- return {"services": services, "needs_update": needs_update}
+ unknown = any(s["up_to_date"] is None for s in services.values())
+ return {"services": services, "needs_update": needs_update, "unknown": unknown}
async def update_customer_containers(instance_dir: str, project_name: str) -> dict[str, Any]:
@@ -292,6 +385,13 @@ async def update_customer_containers(instance_dir: str, project_name: str) -> di
compose_file = os.path.join(instance_dir, "docker-compose.yml")
if not os.path.isfile(compose_file):
return {"success": False, "error": f"docker-compose.yml not found at {compose_file}"}
+
+ # Repair any container still orphaned under a hash-prefixed name from a
+ # previous interrupted recreate before Compose tries to touch it again —
+ # otherwise Compose keeps colliding with the same stuck rename.
+ loop = asyncio.get_event_loop()
+ await loop.run_in_executor(None, repair_container_naming, project_name)
+
cmd = [
"docker", "compose",
"-f", compose_file,
diff --git a/app/services/scheduler_service.py b/app/services/scheduler_service.py
new file mode 100644
index 0000000..6722991
--- /dev/null
+++ b/app/services/scheduler_service.py
@@ -0,0 +1,119 @@
+"""Background scheduler for automatic NetBird image update checks.
+
+No external scheduler dependency (APScheduler etc.) — a single asyncio task
+started at app startup wakes up once a minute, and only actually does
+anything once per day at the configured HH:MM, controlled entirely by
+SystemConfig.auto_update_check_enabled / auto_update_check_time.
+"""
+
+import asyncio
+import logging
+from datetime import datetime
+
+from app.database import SessionLocal
+from app.models import Deployment, SystemConfig
+from app.services import image_service
+
+logger = logging.getLogger(__name__)
+
+_POLL_INTERVAL_SECONDS = 60
+_task: asyncio.Task | None = None
+
+
+def start() -> None:
+ """Start the background polling task. Safe to call once at app startup."""
+ global _task
+ if _task is None or _task.done():
+ _task = asyncio.create_task(_poll_loop())
+ logger.info("Automatic update scheduler started.")
+
+
+def stop() -> None:
+ """Cancel the background polling task."""
+ global _task
+ if _task is not None:
+ _task.cancel()
+ _task = None
+
+
+async def _poll_loop() -> None:
+ while True:
+ try:
+ await _tick()
+ except Exception:
+ logger.exception("Scheduler tick failed")
+ await asyncio.sleep(_POLL_INTERVAL_SECONDS)
+
+
+async def _tick() -> None:
+ db = SessionLocal()
+ try:
+ config = db.query(SystemConfig).filter(SystemConfig.id == 1).first()
+ if not config or not config.auto_update_check_enabled:
+ return
+
+ now = datetime.now()
+ target_time = config.auto_update_check_time or "03:00"
+ current_hhmm = now.strftime("%H:%M")
+ if current_hhmm != target_time:
+ return
+
+ last_run = config.auto_update_last_run_at
+ if last_run and last_run.date() == now.date():
+ return # already ran today
+
+ # Claim this run immediately so a slow run can't overlap the next tick.
+ config.auto_update_last_run_at = now
+ db.commit()
+
+ apply_enabled = bool(config.auto_update_apply_enabled)
+ logger.info(
+ "Running scheduled NetBird image update check (auto-apply=%s)...", apply_enabled
+ )
+ await _run_check_and_optionally_apply(config, apply_enabled)
+ finally:
+ db.close()
+
+
+async def _run_check_and_optionally_apply(config: SystemConfig, apply_enabled: bool) -> None:
+ hub_status = await image_service.check_all_images(config)
+ if not hub_status["any_update_available"]:
+ logger.info("Scheduled check: all NetBird images already up to date.")
+ return
+
+ logger.info("Scheduled check: new NetBird image(s) available — pulling.")
+ pull_result = await image_service.pull_all_images(config)
+ if not pull_result["all_success"]:
+ logger.error("Scheduled image pull had failures: %s", pull_result["results"])
+
+ if not apply_enabled:
+ logger.info("Auto-apply disabled — images pulled, customer containers left untouched.")
+ return
+
+ db = SessionLocal()
+ try:
+ deployments = db.query(Deployment).all()
+ to_update = []
+ for dep in deployments:
+ cs = image_service.get_customer_container_image_status(dep.container_prefix, config)
+ if cs["needs_update"]:
+ customer = dep.customer
+ to_update.append({
+ "instance_dir": f"{config.data_dir}/{customer.subdomain}",
+ "project_name": dep.container_prefix,
+ "customer_name": customer.name,
+ })
+ logger.info("Scheduled auto-apply: updating %d customer(s)...", len(to_update))
+ for entry in to_update:
+ try:
+ res = await image_service.update_customer_containers(
+ entry["instance_dir"], entry["project_name"]
+ )
+ logger.info(
+ "Scheduled update for %s: %s",
+ entry["customer_name"], "OK" if res["success"] else res.get("error"),
+ )
+ except Exception:
+ logger.exception("Scheduled update failed for %s", entry["customer_name"])
+ finally:
+ db.close()
diff --git a/app/utils/validators.py b/app/utils/validators.py
index 27c6181..686015c 100644
--- a/app/utils/validators.py
+++ b/app/utils/validators.py
@@ -158,6 +158,21 @@ class SystemConfigUpdate(BaseModel):
git_repo_url: Optional[str] = Field(None, max_length=500)
git_branch: Optional[str] = Field(None, max_length=100)
git_token: Optional[str] = None # plaintext, encrypted before storage
+ # Automatic NetBird image update check/apply
+ auto_update_check_enabled: Optional[bool] = None
+ auto_update_check_time: Optional[str] = Field(None, max_length=5)
+ auto_update_apply_enabled: Optional[bool] = None
+
+ @field_validator("auto_update_check_time")
+ @classmethod
+ def validate_auto_update_check_time(cls, v: Optional[str]) -> Optional[str]:
+ """Must be HH:MM in 24h format."""
+ if v is None:
+ return v
+ import re
+ if not re.fullmatch(r"([01]\d|2[0-3]):[0-5]\d", v):
+ raise ValueError("auto_update_check_time must be in HH:MM 24h format")
+ return v
@field_validator("ssl_mode")
@classmethod
diff --git a/static/index.html b/static/index.html
index 8861e50..afb6d8d 100644
--- a/static/index.html
+++ b/static/index.html
@@ -641,6 +641,33 @@
Pull from Docker Hub
+
+ Automatic Updates
+ Automatically checks for new NetBird images daily at the chosen time.
+
diff --git a/static/js/app.js b/static/js/app.js
index a8b9fed..49301a5 100644
--- a/static/js/app.js
+++ b/static/js/app.js
@@ -940,6 +940,13 @@ async function loadSettings() {
document.getElementById('cfg-relay-image').value = cfg.netbird_relay_image || '';
document.getElementById('cfg-dashboard-image').value = cfg.netbird_dashboard_image || '';
+ document.getElementById('cfg-auto-update-check-enabled').checked = cfg.auto_update_check_enabled || false;
+ document.getElementById('cfg-auto-update-check-time').value = cfg.auto_update_check_time || '03:00';
+ document.getElementById('cfg-auto-update-apply-enabled').checked = cfg.auto_update_apply_enabled || false;
+ document.getElementById('auto-update-last-run').textContent = cfg.auto_update_last_run_at
+ ? new Date(cfg.auto_update_last_run_at).toLocaleString()
+ : t('monitoring.autoUpdateNever');
+
// Branding tab
document.getElementById('cfg-branding-name').value = cfg.branding_name || '';
document.getElementById('cfg-branding-subtitle').value = cfg.branding_subtitle || '';
@@ -1058,6 +1065,21 @@ document.getElementById('settings-images-form').addEventListener('submit', async
}
});
+// Automatic update settings form
+document.getElementById('settings-auto-update-form').addEventListener('submit', async (e) => {
+ e.preventDefault();
+ try {
+ await api('PUT', '/settings/system', {
+ auto_update_check_enabled: document.getElementById('cfg-auto-update-check-enabled').checked,
+ auto_update_check_time: document.getElementById('cfg-auto-update-check-time').value || '03:00',
+ auto_update_apply_enabled: document.getElementById('cfg-auto-update-apply-enabled').checked,
+ });
+ showSettingsAlert('success', t('messages.imageSettingsSaved'));
+ } catch (err) {
+ showSettingsAlert('danger', t('errors.failed', { error: err.message }));
+ }
+});
+
// Test NPM connection
async function testNpmConnection() {
const spinner = document.getElementById('npm-test-spinner');
@@ -1807,7 +1829,9 @@ async function checkImageUpdates() {
: data.customer_status.map(c => {
const badge = c.needs_update
? `${t('monitoring.needsUpdate')}`
- : `${t('monitoring.upToDate')}`;
+ : c.unknown
+ ? `${t('monitoring.statusUnknown')}`
+ : `${t('monitoring.upToDate')}`;
const updateBtn = c.needs_update
? ``
diff --git a/static/lang/de.json b/static/lang/de.json
index 24b1e16..4b02698 100644
--- a/static/lang/de.json
+++ b/static/lang/de.json
@@ -420,6 +420,17 @@
"updating": "Wird aktualisiert…",
"updateAllProgress": "Kunden-Container werden nacheinander aktualisiert — bitte warten…",
"pulling": "Wird geladen…",
- "pullStartedShort": "Download im Hintergrund gestartet."
+ "pullStartedShort": "Download im Hintergrund gestartet.",
+ "statusUnknown": "Unbekannt",
+ "statusUnknownHint": "Container konnte nicht eindeutig zugeordnet werden (z. B. nicht gestartet). Kein verifiziertes \"Aktuell\".",
+ "autoUpdateTitle": "Automatische Aktualisierung",
+ "autoUpdateHint": "Prüft täglich zur gewählten Uhrzeit automatisch auf neue NetBird-Images.",
+ "autoUpdateCheckEnabled": "Automatische Update-Prüfung aktivieren",
+ "autoUpdateCheckTime": "Uhrzeit der täglichen Prüfung",
+ "autoUpdateApplyEnabled": "Kunden-Container nach Prüfung automatisch aktualisieren",
+ "autoUpdateApplyHint": "Wenn aktiviert, werden nach einer gefundenen Aktualisierung automatisch alle Kunden-Container neu erstellt (kurzer Neustart der Dienste). Wenn deaktiviert, werden nur die Images geladen — die Aktualisierung der Kunden erfolgt weiterhin manuell.",
+ "autoUpdateLastRun": "Letzte automatische Prüfung",
+ "autoUpdateNever": "Noch nie ausgeführt",
+ "saveAutoUpdateSettings": "Automatisierung speichern"
}
}
\ No newline at end of file
diff --git a/static/lang/en.json b/static/lang/en.json
index ae03a28..291dc07 100644
--- a/static/lang/en.json
+++ b/static/lang/en.json
@@ -327,7 +327,18 @@
"updating": "Updating…",
"updateAllProgress": "Updating customer containers one by one — please wait…",
"pulling": "Pulling…",
- "pullStartedShort": "Pull started in background."
+ "pullStartedShort": "Pull started in background.",
+ "statusUnknown": "Unknown",
+ "statusUnknownHint": "Container could not be matched reliably (e.g. not running). Not a verified \"up to date\".",
+ "autoUpdateTitle": "Automatic Updates",
+ "autoUpdateHint": "Automatically checks for new NetBird images daily at the chosen time.",
+ "autoUpdateCheckEnabled": "Enable automatic update check",
+ "autoUpdateCheckTime": "Daily check time",
+ "autoUpdateApplyEnabled": "Automatically update customer containers after check",
+ "autoUpdateApplyHint": "When enabled, all customer containers are automatically recreated after a new image is found (services briefly restart). When disabled, only the images are pulled — updating customers stays a manual step.",
+ "autoUpdateLastRun": "Last automatic check",
+ "autoUpdateNever": "Never run",
+ "saveAutoUpdateSettings": "Save Automation"
},
"userModal": {
"title": "New User",