Compare commits

...
8 Commits
Author SHA1 Message Date
twothatIT 556db5bfd4 chore(release): bump version to 1.5.0 2026-08-21 22:40:22 +02:00
twothatIT d4b6efeea1 feat(monitoring): add update-run log, fix scheduler skipping stale customer containers
The automatic scheduler only recreated customer containers when it had just
found a NEW image on Docker Hub. If an image was already pulled locally
(e.g. from an earlier manual pull, or auto-apply was off at the time) but
customer containers hadn't been recreated yet, the daily check reported
"already up to date" and silently never touched those customers again —
the recreate step needs to run independently of whether Hub had anything
new, based on each customer's actual container-vs-local-image state.

Also adds a persistent UpdateRunLog: every scheduled and manual pull/update
run now records per-image pull results and per-customer recreate results,
visible in Monitoring > Update Log, instead of only existing in container
logs.
2026-08-21 22:40:12 +02:00
twothatIT 276f4214a7 chore(release): bump version to 1.4.2 2026-08-19 11:22:08 +02:00
twothatIT a8ed87faa9 feat(customer): show NetBird API token renewal/expiry date
Displays when the per-customer token was last (re-)issued and its
computed expiry (365 days from renewal) on the Auto-Updates card, so it's
visible at a glance without checking the database.
2026-08-19 11:21:55 +02:00
twothatIT 1bc768151a feat(customer): allow changing an already-registered NetBird API token
The per-customer Auto-Updates card only showed the token input when no
token was registered yet, so there was no way to replace one once set
(e.g. rotating it manually). Adds a "Change token" link that reveals the
same input/save flow at any time.
2026-08-19 11:21:10 +02:00
twothatIT 1e05ddfe69 chore(release): bump version to 1.4.1 2026-08-19 11:17:38 +02:00
twothatIT e9afdb226c feat(netbird): auto-renew NetBird client-update API tokens before expiry
The tokens captured for central update control expire (NetBird enforces a
365-day max on Personal Access Tokens), and nothing was renewing them —
discovered that the 47 tokens created via the browser-automation bulk
onboarding were actually only 30-day tokens (left the UI's default
expiration field untouched instead of setting 365), so they would have
silently broken automatic-update control next month with no warning.

- Bumped all existing tokens to fresh 365-day ones via the API (using the
  still-valid old token as bearer — no re-login needed)
- Added netbird_api_token_renewed_at per deployment
- Scheduler now checks daily and renews any token older than 300 days
  automatically, so this never needs to be done by hand again
2026-08-19 11:15:49 +02:00
twothatIT 8a84e60a27 fix(validators): restore SystemConfigUpdate validators broken by misplaced classes
The two new NetbirdClientAutoUpdatePayload/NetbirdApiTokenPayload classes
were accidentally inserted between two of SystemConfigUpdate's own
@field_validator methods, which closed that class early — the ssl_mode,
base_domain, npm_api_url, and admin_email validators ended up attached to
NetbirdApiTokenPayload instead, whose fields don't exist. Pydantic raised
PydanticUserError on import, crash-looping the whole appliance container.
Moved the two new classes after all of SystemConfigUpdate's validators.
2026-08-19 09:35:12 +02:00
14 changed files with 558 additions and 60 deletions
+2
View File
@@ -45,6 +45,7 @@ def init_db() -> None:
Deployment, Deployment,
DeploymentLog, DeploymentLog,
SystemConfig, SystemConfig,
UpdateRunLog,
User, User,
) )
@@ -129,6 +130,7 @@ def _run_migrations() -> None:
("system_config", "auto_update_last_run_at", "TEXT"), ("system_config", "auto_update_last_run_at", "TEXT"),
# NetBird client (peer) automatic-updates master default + per-customer PAT # NetBird client (peer) automatic-updates master default + per-customer PAT
("deployments", "netbird_api_token_encrypted", "TEXT"), ("deployments", "netbird_api_token_encrypted", "TEXT"),
("deployments", "netbird_api_token_renewed_at", "TEXT"),
("system_config", "netbird_client_auto_update_version", "TEXT DEFAULT 'disabled'"), ("system_config", "netbird_client_auto_update_version", "TEXT DEFAULT 'disabled'"),
("system_config", "netbird_client_auto_update_always", "BOOLEAN DEFAULT 0"), ("system_config", "netbird_client_auto_update_always", "BOOLEAN DEFAULT 0"),
] ]
+1 -1
View File
@@ -34,7 +34,7 @@ logger = logging.getLogger(__name__)
app = FastAPI( app = FastAPI(
title="NetBird MSP Appliance", title="NetBird MSP Appliance",
description="Multi-tenant NetBird management platform for MSPs", description="Multi-tenant NetBird management platform for MSPs",
version="1.4.0", version="1.5.0",
docs_url="/api/docs", docs_url="/api/docs",
redoc_url="/api/redoc", redoc_url="/api/redoc",
openapi_url="/api/openapi.json", openapi_url="/api/openapi.json",
+55
View File
@@ -1,5 +1,6 @@
"""SQLAlchemy ORM models for NetBird MSP Appliance.""" """SQLAlchemy ORM models for NetBird MSP Appliance."""
import json
from datetime import datetime from datetime import datetime
from typing import Optional from typing import Optional
@@ -89,6 +90,7 @@ class Deployment(Base):
netbird_admin_email: Mapped[Optional[str]] = mapped_column(Text, nullable=True) netbird_admin_email: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
netbird_admin_password: Mapped[Optional[str]] = mapped_column(Text, nullable=True) netbird_admin_password: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
netbird_api_token_encrypted: Mapped[Optional[str]] = mapped_column(Text, nullable=True) netbird_api_token_encrypted: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
netbird_api_token_renewed_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
deployment_status: Mapped[str] = mapped_column( deployment_status: Mapped[str] = mapped_column(
String(20), default="pending", nullable=False String(20), default="pending", nullable=False
) )
@@ -118,6 +120,9 @@ class Deployment(Base):
"setup_url": self.setup_url, "setup_url": self.setup_url,
"has_credentials": bool(self.netbird_admin_email and self.netbird_admin_password), "has_credentials": bool(self.netbird_admin_email and self.netbird_admin_password),
"has_netbird_api_token": bool(self.netbird_api_token_encrypted), "has_netbird_api_token": bool(self.netbird_api_token_encrypted),
"netbird_api_token_renewed_at": (
self.netbird_api_token_renewed_at.isoformat() if self.netbird_api_token_renewed_at else None
),
"deployment_status": self.deployment_status, "deployment_status": self.deployment_status,
"deployed_at": self.deployed_at.isoformat() if self.deployed_at else None, "deployed_at": self.deployed_at.isoformat() if self.deployed_at else None,
"last_health_check": ( "last_health_check": (
@@ -317,6 +322,56 @@ class DeploymentLog(Base):
} }
class UpdateRunLog(Base):
"""Audit trail for NetBird Docker image update runs (pull + container recreate).
Written for both scheduled (automatic) and manual (button-triggered)
runs, so the actual outcome — which image pulls failed, which customer
container recreates failed — is visible in the UI instead of only in
container logs.
"""
__tablename__ = "update_run_logs"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
run_type: Mapped[str] = mapped_column(String(20), nullable=False) # "scheduled" | "manual"
trigger: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) # username, or "scheduler"
started_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
finished_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
status: Mapped[str] = mapped_column(
String(20), default="running"
) # running | no_update | success | partial | failed
any_update_available: Mapped[Optional[bool]] = mapped_column(Boolean, nullable=True)
pull_attempted: Mapped[bool] = mapped_column(Boolean, default=False)
pull_success: Mapped[Optional[bool]] = mapped_column(Boolean, nullable=True)
pull_details: Mapped[Optional[str]] = mapped_column(Text, nullable=True) # JSON: {image: {success, error}}
apply_attempted: Mapped[bool] = mapped_column(Boolean, default=False)
customers_updated: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
customers_total: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
customer_details: Mapped[Optional[str]] = mapped_column(Text, nullable=True) # JSON list
error: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
def to_dict(self) -> dict:
"""Serialize to dict, parsing the embedded JSON blobs."""
return {
"id": self.id,
"run_type": self.run_type,
"trigger": self.trigger,
"started_at": self.started_at.isoformat() if self.started_at else None,
"finished_at": self.finished_at.isoformat() if self.finished_at else None,
"status": self.status,
"any_update_available": self.any_update_available,
"pull_attempted": self.pull_attempted,
"pull_success": self.pull_success,
"pull_details": json.loads(self.pull_details) if self.pull_details else None,
"apply_attempted": self.apply_attempted,
"customers_updated": self.customers_updated,
"customers_total": self.customers_total,
"customer_details": json.loads(self.customer_details) if self.customer_details else None,
"error": self.error,
}
class User(Base): class User(Base):
"""Admin user model.""" """Admin user model."""
+16 -1
View File
@@ -1,6 +1,7 @@
"""Deployment management API — start, stop, restart, logs, health for customers.""" """Deployment management API — start, stop, restart, logs, health for customers."""
import logging import logging
from datetime import datetime
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query, status from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query, status
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
@@ -8,7 +9,7 @@ from sqlalchemy.orm import Session
from app.database import SessionLocal, get_db from app.database import SessionLocal, get_db
from app.dependencies import get_current_user from app.dependencies import get_current_user
from app.models import Customer, Deployment, SystemConfig, User from app.models import Customer, Deployment, SystemConfig, User
from app.services import docker_service, image_service, netbird_client_update_service, netbird_service from app.services import docker_service, image_service, netbird_client_update_service, netbird_service, update_log_service
from app.utils.security import decrypt_value, encrypt_value from app.utils.security import decrypt_value, encrypt_value
from app.utils.validators import NetbirdApiTokenPayload, NetbirdClientAutoUpdatePayload from app.utils.validators import NetbirdApiTokenPayload, NetbirdClientAutoUpdatePayload
@@ -245,9 +246,18 @@ async def update_customer_images(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="System not configured." status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="System not configured."
) )
started_at = datetime.utcnow()
instance_dir = f"{config.data_dir}/{customer.subdomain}" instance_dir = f"{config.data_dir}/{customer.subdomain}"
result = await image_service.update_customer_containers(instance_dir, deployment.container_prefix) result = await image_service.update_customer_containers(instance_dir, deployment.container_prefix)
update_log_service.record_run(
db, run_type="manual", trigger=current_user.username, started_at=started_at,
apply_attempted=True,
customer_results=[{
"customer_name": customer.name, "success": result["success"], "error": result.get("error"),
}],
)
if not result["success"]: if not result["success"]:
raise HTTPException( raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
@@ -287,6 +297,10 @@ async def get_customer_netbird_updates(
settings = result["settings"] settings = result["settings"]
return { return {
"has_token": True, "has_token": True,
"token_renewed_at": (
deployment.netbird_api_token_renewed_at.isoformat()
if deployment.netbird_api_token_renewed_at else None
),
"version": settings.get("auto_update_version", "disabled"), "version": settings.get("auto_update_version", "disabled"),
"always": bool(settings.get("auto_update_always", False)), "always": bool(settings.get("auto_update_always", False)),
} }
@@ -362,6 +376,7 @@ async def set_customer_netbird_api_token(
) )
deployment.netbird_api_token_encrypted = encrypt_value(payload.token) deployment.netbird_api_token_encrypted = encrypt_value(payload.token)
deployment.netbird_api_token_renewed_at = datetime.utcnow()
db.commit() db.commit()
logger.info("NetBird API token registered for customer %d by %s.", customer_id, current_user.username) logger.info("NetBird API token registered for customer %d by %s.", customer_id, current_user.username)
return {"ok": True} return {"ok": True}
+48 -5
View File
@@ -4,6 +4,7 @@ import asyncio
import logging import logging
import platform import platform
import time import time
from datetime import datetime
from typing import Any from typing import Any
import psutil import psutil
@@ -12,8 +13,8 @@ from sqlalchemy.orm import Session
from app.database import SessionLocal, get_db from app.database import SessionLocal, get_db
from app.dependencies import get_current_user from app.dependencies import get_current_user
from app.models import Customer, Deployment, SystemConfig, User from app.models import Customer, Deployment, SystemConfig, UpdateRunLog, User
from app.services import docker_service, image_service, netbird_client_update_service from app.services import docker_service, image_service, netbird_client_update_service, update_log_service
from app.utils.security import decrypt_value from app.utils.security import decrypt_value
from app.utils.validators import NetbirdClientAutoUpdatePayload from app.utils.validators import NetbirdClientAutoUpdatePayload
@@ -194,14 +195,25 @@ async def pull_all_netbird_images(
config.netbird_dashboard_image, config.netbird_dashboard_image,
] ]
username = current_user.username
async def _pull_bg() -> None: async def _pull_bg() -> None:
started_at = datetime.utcnow()
bg_db = SessionLocal() bg_db = SessionLocal()
try: try:
cfg = bg_db.query(SystemConfig).filter(SystemConfig.id == 1).first() cfg = bg_db.query(SystemConfig).filter(SystemConfig.id == 1).first()
if cfg: if not cfg:
await image_service.pull_all_images(cfg) return
except Exception: pull_result = await image_service.pull_all_images(cfg)
update_log_service.record_run(
bg_db, run_type="manual", trigger=username, started_at=started_at,
any_update_available=True, pull_attempted=True, pull_results=pull_result["results"],
)
except Exception as exc:
logger.exception("Background image pull failed") logger.exception("Background image pull failed")
update_log_service.record_run(
bg_db, run_type="manual", trigger=username, started_at=started_at, error=str(exc),
)
finally: finally:
bg_db.close() bg_db.close()
@@ -285,6 +297,28 @@ async def apply_netbird_client_updates_to_all(
} }
@router.get("/update-logs")
async def get_update_logs(
limit: int = 30,
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
) -> list[dict[str, Any]]:
"""Return the most recent NetBird image update runs (scheduled and manual).
Shows exactly what happened on each run: whether an image pull was
attempted and its per-image result, and whether customer containers
were recreated and which ones failed — so this doesn't have to be
reconstructed from container logs.
"""
logs = (
db.query(UpdateRunLog)
.order_by(UpdateRunLog.started_at.desc())
.limit(min(limit, 100))
.all()
)
return [log.to_dict() for log in logs]
@router.post("/customers/update-all") @router.post("/customers/update-all")
async def update_all_customers( async def update_all_customers(
current_user: User = Depends(get_current_user), current_user: User = Depends(get_current_user),
@@ -299,6 +333,7 @@ async def update_all_customers(
if current_user.role != "admin": if current_user.role != "admin":
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin only.") raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin only.")
started_at = datetime.utcnow()
config = db.query(SystemConfig).filter(SystemConfig.id == 1).first() config = db.query(SystemConfig).filter(SystemConfig.id == 1).first()
if not config: if not config:
raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="System not configured.") raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="System not configured.")
@@ -317,6 +352,10 @@ async def update_all_customers(
}) })
if not to_update: if not to_update:
update_log_service.record_run(
db, run_type="manual", trigger=current_user.username,
started_at=started_at, apply_attempted=True, customer_results=[],
)
return {"message": "All customers are already up to date.", "updated": 0, "results": []} return {"message": "All customers are already up to date.", "updated": 0, "results": []}
# Update customers sequentially — one at a time. A failure for one # Update customers sequentially — one at a time. A failure for one
@@ -343,6 +382,10 @@ async def update_all_customers(
}) })
success_count = sum(1 for r in update_results if r["success"]) success_count = sum(1 for r in update_results if r["success"])
update_log_service.record_run(
db, run_type="manual", trigger=current_user.username,
started_at=started_at, apply_attempted=True, customer_results=update_results,
)
return { return {
"message": f"Updated {success_count} of {len(update_results)} customer(s).", "message": f"Updated {success_count} of {len(update_results)} customer(s).",
"updated": success_count, "updated": success_count,
@@ -52,6 +52,48 @@ async def get_current_settings(container_prefix: str, token: str) -> dict[str, A
return {"ok": False, "error": str(exc)} return {"ok": False, "error": str(exc)}
async def renew_token(container_prefix: str, token: str) -> dict[str, Any]:
"""Mint a fresh 365-day Personal Access Token using the current one.
The old token is left in place (it naturally expires and NetBird gives no
reliable way to identify "our" token among a user's other PATs by content
alone, only by name — which isn't safe to assume is unique). One
harmless unused token lingering until its own expiry is an acceptable
trade-off for not risking deleting a token that turns out to be in use.
"""
base_url = _base_url(container_prefix)
headers = {"Authorization": f"Token {token}"}
try:
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
resp = await client.get(f"{base_url}/api/users", headers=headers)
if resp.status_code != 200:
return {"ok": False, "error": f"GET /api/users -> HTTP {resp.status_code}"}
users = resp.json()
me = next((u for u in users if u.get("is_current")), None)
if not me:
return {"ok": False, "error": "Could not identify current user from /api/users."}
create_resp = await client.post(
f"{base_url}/api/users/{me['id']}/tokens",
headers={**headers, "Content-Type": "application/json"},
content=json.dumps({"name": "MSP Central Management", "expires_in": 365}),
)
if create_resp.status_code not in (200, 201):
return {"ok": False, "error": f"POST tokens -> HTTP {create_resp.status_code}: {create_resp.text[:200]}"}
new_token = create_resp.json().get("plain_token")
if not new_token:
return {"ok": False, "error": "Token creation response had no plain_token."}
verify_resp = await client.get(f"{base_url}/api/accounts", headers={"Authorization": f"Token {new_token}"})
if verify_resp.status_code != 200:
return {"ok": False, "error": "New token failed verification."}
return {"ok": True, "token": new_token}
except Exception as exc:
logger.warning("Failed to renew NetBird API token for %s: %s", container_prefix, exc)
return {"ok": False, "error": str(exc)}
async def push_auto_update_settings( async def push_auto_update_settings(
container_prefix: str, token: str, version: str, always: bool container_prefix: str, token: str, version: str, always: bool
) -> dict[str, Any]: ) -> dict[str, Any]:
+4 -3
View File
@@ -348,9 +348,9 @@ async def deploy_customer(db: Session, customer_id: int) -> dict[str, Any]:
deployment.setup_url = setup_url deployment.setup_url = setup_url
deployment.netbird_admin_email = encrypt_value(admin_email) if setup_ok else deployment.netbird_admin_email deployment.netbird_admin_email = encrypt_value(admin_email) if setup_ok else deployment.netbird_admin_email
deployment.netbird_admin_password = encrypt_value(admin_password) if setup_ok else deployment.netbird_admin_password deployment.netbird_admin_password = encrypt_value(admin_password) if setup_ok else deployment.netbird_admin_password
deployment.netbird_api_token_encrypted = ( if netbird_api_token:
encrypt_value(netbird_api_token) if netbird_api_token else deployment.netbird_api_token_encrypted deployment.netbird_api_token_encrypted = encrypt_value(netbird_api_token)
) deployment.netbird_api_token_renewed_at = datetime.utcnow()
deployment.deployment_status = "running" deployment.deployment_status = "running"
deployment.deployed_at = datetime.utcnow() deployment.deployed_at = datetime.utcnow()
else: else:
@@ -366,6 +366,7 @@ async def deploy_customer(db: Session, customer_id: int) -> dict[str, Any]:
netbird_admin_email=encrypt_value(admin_email) if setup_ok else None, netbird_admin_email=encrypt_value(admin_email) if setup_ok else None,
netbird_admin_password=encrypt_value(admin_password) if setup_ok else None, netbird_admin_password=encrypt_value(admin_password) if setup_ok else None,
netbird_api_token_encrypted=encrypt_value(netbird_api_token) if netbird_api_token else None, netbird_api_token_encrypted=encrypt_value(netbird_api_token) if netbird_api_token else None,
netbird_api_token_renewed_at=datetime.utcnow() if netbird_api_token else None,
deployment_status="running", deployment_status="running",
deployed_at=datetime.utcnow(), deployed_at=datetime.utcnow(),
) )
+116 -25
View File
@@ -8,17 +8,23 @@ SystemConfig.auto_update_check_enabled / auto_update_check_time.
import asyncio import asyncio
import logging import logging
from datetime import datetime from datetime import datetime, timedelta
from app.database import SessionLocal from app.database import SessionLocal
from app.models import Deployment, SystemConfig from app.models import Deployment, SystemConfig
from app.services import image_service from app.services import image_service, netbird_client_update_service, update_log_service
from app.utils.security import decrypt_value, encrypt_value
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
_POLL_INTERVAL_SECONDS = 60 _POLL_INTERVAL_SECONDS = 60
_task: asyncio.Task | None = None _task: asyncio.Task | None = None
# NetBird PATs we mint are issued for 365 days; renew well before that so a
# missed tick or a slow rollout never risks the token actually expiring.
_TOKEN_RENEW_AFTER_DAYS = 300
_TOKEN_RENEW_CHECK_HOUR = 4 # run once per day, distinct from the image-check hour
def start() -> None: def start() -> None:
"""Start the background polling task. Safe to call once at app startup.""" """Start the background polling task. Safe to call once at app startup."""
@@ -36,15 +42,64 @@ def stop() -> None:
_task = None _task = None
_last_token_renewal_date = None
async def _poll_loop() -> None: async def _poll_loop() -> None:
while True: while True:
try: try:
await _tick() await _tick()
except Exception: except Exception:
logger.exception("Scheduler tick failed") logger.exception("Scheduler tick failed")
try:
await _token_renewal_tick()
except Exception:
logger.exception("Token renewal tick failed")
await asyncio.sleep(_POLL_INTERVAL_SECONDS) await asyncio.sleep(_POLL_INTERVAL_SECONDS)
async def _token_renewal_tick() -> None:
"""Once a day, renew any NetBird client-update API token nearing its
365-day expiry — keeps central update control working indefinitely
without anyone needing to notice or act.
"""
global _last_token_renewal_date
now = datetime.now()
if now.hour != _TOKEN_RENEW_CHECK_HOUR:
return
if _last_token_renewal_date == now.date():
return
_last_token_renewal_date = now.date()
db = SessionLocal()
try:
cutoff = now - timedelta(days=_TOKEN_RENEW_AFTER_DAYS)
deployments = (
db.query(Deployment)
.filter(Deployment.netbird_api_token_encrypted.isnot(None))
.all()
)
due = [
d for d in deployments
if d.netbird_api_token_renewed_at is None or d.netbird_api_token_renewed_at < cutoff
]
if not due:
return
logger.info("Renewing NetBird API token for %d customer(s)...", len(due))
for d in due:
token = decrypt_value(d.netbird_api_token_encrypted)
result = await netbird_client_update_service.renew_token(d.container_prefix, token)
if result["ok"]:
d.netbird_api_token_encrypted = encrypt_value(result["token"])
d.netbird_api_token_renewed_at = now
db.commit()
logger.info("Renewed NetBird API token for %s.", d.container_prefix)
else:
logger.warning("Failed to renew NetBird API token for %s: %s", d.container_prefix, result.get("error"))
finally:
db.close()
async def _tick() -> None: async def _tick() -> None:
db = SessionLocal() db = SessionLocal()
try: try:
@@ -70,29 +125,49 @@ async def _tick() -> None:
logger.info( logger.info(
"Running scheduled NetBird image update check (auto-apply=%s)...", apply_enabled "Running scheduled NetBird image update check (auto-apply=%s)...", apply_enabled
) )
await _run_check_and_optionally_apply(config, apply_enabled) await _run_check_and_optionally_apply(config, apply_enabled, started_at=now)
finally: finally:
db.close() db.close()
async def _run_check_and_optionally_apply(config: SystemConfig, apply_enabled: bool) -> None: async def _run_check_and_optionally_apply(config: SystemConfig, apply_enabled: bool, started_at: datetime) -> None:
hub_status = await image_service.check_all_images(config) """Check Docker Hub, pull if needed, then (if enabled) recreate any
if not hub_status["any_update_available"]: customer container that isn't running the locally-pulled image yet.
logger.info("Scheduled check: all NetBird images already up to date.")
return
logger.info("Scheduled check: new NetBird image(s) available — pulling.") The "recreate" step is intentionally NOT gated on a Hub update having
pull_result = await image_service.pull_all_images(config) just been found: a customer can already be behind the *locally pulled*
if not pull_result["all_success"]: image from an earlier check/pull that was never applied (e.g. auto-apply
logger.error("Scheduled image pull had failures: %s", pull_result["results"]) was off at the time, or a previous run failed) — that backlog must keep
being cleared on every run, not just on runs that also found something
if not apply_enabled: new on Hub, otherwise it silently never happens.
logger.info("Auto-apply disabled — images pulled, customer containers left untouched.") """
return log_db = SessionLocal()
db = SessionLocal()
try: try:
deployments = db.query(Deployment).all() hub_status = await image_service.check_all_images(config)
any_update = hub_status["any_update_available"]
pull_attempted = False
pull_results = None
if any_update:
logger.info("Scheduled check: new NetBird image(s) available — pulling.")
pull_attempted = True
pull_result = await image_service.pull_all_images(config)
pull_results = pull_result["results"]
if not pull_result["all_success"]:
logger.error("Scheduled image pull had failures: %s", pull_results)
else:
logger.info("Scheduled check: all NetBird images already up to date on Docker Hub.")
if not apply_enabled:
logger.info("Auto-apply disabled — customer containers left untouched.")
update_log_service.record_run(
log_db, run_type="scheduled", trigger="scheduler",
started_at=started_at, any_update_available=any_update,
pull_attempted=pull_attempted, pull_results=pull_results,
)
return
deployments = log_db.query(Deployment).all()
to_update = [] to_update = []
for dep in deployments: for dep in deployments:
cs = image_service.get_customer_container_image_status(dep.container_prefix, config) cs = image_service.get_customer_container_image_status(dep.container_prefix, config)
@@ -104,16 +179,32 @@ async def _run_check_and_optionally_apply(config: SystemConfig, apply_enabled: b
"customer_name": customer.name, "customer_name": customer.name,
}) })
logger.info("Scheduled auto-apply: updating %d customer(s)...", len(to_update)) logger.info("Scheduled auto-apply: updating %d customer(s)...", len(to_update))
customer_results = []
for entry in to_update: for entry in to_update:
try: try:
res = await image_service.update_customer_containers( res = await image_service.update_customer_containers(
entry["instance_dir"], entry["project_name"] entry["instance_dir"], entry["project_name"]
) )
logger.info( ok = res["success"]
"Scheduled update for %s: %s", err = res.get("error")
entry["customer_name"], "OK" if res["success"] else res.get("error"), except Exception as exc:
)
except Exception:
logger.exception("Scheduled update failed for %s", entry["customer_name"]) logger.exception("Scheduled update failed for %s", entry["customer_name"])
ok = False
err = str(exc)
logger.info("Scheduled update for %s: %s", entry["customer_name"], "OK" if ok else err)
customer_results.append({"customer_name": entry["customer_name"], "success": ok, "error": err})
update_log_service.record_run(
log_db, run_type="scheduled", trigger="scheduler",
started_at=started_at, any_update_available=any_update,
pull_attempted=pull_attempted, pull_results=pull_results,
apply_attempted=True, customer_results=customer_results,
)
except Exception as exc:
logger.exception("Scheduled update run failed")
update_log_service.record_run(
log_db, run_type="scheduled", trigger="scheduler",
started_at=started_at, error=str(exc),
)
finally: finally:
db.close() log_db.close()
+84
View File
@@ -0,0 +1,84 @@
"""Persists the outcome of every NetBird Docker image update run.
Both the automatic scheduler and the manual "Check/Pull/Update" buttons in
the UI write here, so it's visible afterwards which run pulled which image
successfully, and which customer container recreate failed — instead of
that information only existing in container logs.
"""
import json
import logging
from datetime import datetime
from typing import Any, Optional
from sqlalchemy.orm import Session
from app.models import UpdateRunLog
logger = logging.getLogger(__name__)
def record_run(
db: Session,
*,
run_type: str,
trigger: Optional[str],
started_at: datetime,
any_update_available: Optional[bool] = None,
pull_attempted: bool = False,
pull_results: Optional[dict[str, Any]] = None,
apply_attempted: bool = False,
customer_results: Optional[list[dict[str, Any]]] = None,
error: Optional[str] = None,
) -> UpdateRunLog:
"""Write one completed update-run entry.
Args:
run_type: "scheduled" or "manual".
trigger: username for manual runs, "scheduler" for automatic ones.
pull_results: {image: {"success": bool, "error": str|None}} as returned
by image_service.pull_all_images()["results"].
customer_results: list of {"customer_name", "success", "error"} as
returned by the update-all / update-customer flows.
"""
pull_success = None
if pull_results:
pull_success = all(r.get("success") for r in pull_results.values())
customers_total = len(customer_results) if customer_results is not None else None
customers_updated = (
sum(1 for r in customer_results if r.get("success")) if customer_results is not None else None
)
if error:
status = "failed"
elif pull_attempted and pull_success is False:
status = "failed"
elif apply_attempted and customers_total and customers_updated != customers_total:
status = "partial"
elif not pull_attempted and not customers_total:
# Nothing to pull (Hub already matched local) and no customer
# container was behind the locally pulled image either.
status = "no_update"
else:
status = "success"
log = UpdateRunLog(
run_type=run_type,
trigger=trigger,
started_at=started_at,
finished_at=datetime.utcnow(),
status=status,
any_update_available=any_update_available,
pull_attempted=pull_attempted,
pull_success=pull_success,
pull_details=json.dumps(pull_results) if pull_results is not None else None,
apply_attempted=apply_attempted,
customers_updated=customers_updated,
customers_total=customers_total,
customer_details=json.dumps(customer_results) if customer_results is not None else None,
error=error,
)
db.add(log)
db.commit()
return log
+21 -21
View File
@@ -177,27 +177,6 @@ class SystemConfigUpdate(BaseModel):
raise ValueError("auto_update_check_time must be in HH:MM 24h format") raise ValueError("auto_update_check_time must be in HH:MM 24h format")
return v return v
# ---------------------------------------------------------------------------
# NetBird client (peer) automatic updates
# ---------------------------------------------------------------------------
class NetbirdClientAutoUpdatePayload(BaseModel):
"""Push a client auto-update version/mode to one or all customers."""
version: str = Field(..., max_length=50, description="'latest', 'disabled', or a version e.g. '0.61.0'")
always: bool = False
class NetbirdApiTokenPayload(BaseModel):
"""Manually register a NetBird Personal Access Token for a customer.
Needed for customers deployed before automatic PAT capture existed —
create a PAT once in that customer's own NetBird dashboard
(Settings > Service Users / Personal Access Tokens) and paste it here.
"""
token: str = Field(..., min_length=10, max_length=500)
@field_validator("ssl_mode") @field_validator("ssl_mode")
@classmethod @classmethod
def validate_ssl_mode(cls, v: Optional[str]) -> Optional[str]: def validate_ssl_mode(cls, v: Optional[str]) -> Optional[str]:
@@ -242,6 +221,27 @@ class NetbirdApiTokenPayload(BaseModel):
return v.lower().strip() return v.lower().strip()
# ---------------------------------------------------------------------------
# NetBird client (peer) automatic updates
# ---------------------------------------------------------------------------
class NetbirdClientAutoUpdatePayload(BaseModel):
"""Push a client auto-update version/mode to one or all customers."""
version: str = Field(..., max_length=50, description="'latest', 'disabled', or a version e.g. '0.61.0'")
always: bool = False
class NetbirdApiTokenPayload(BaseModel):
"""Manually register a NetBird Personal Access Token for a customer.
Needed for customers deployed before automatic PAT capture existed —
create a PAT once in that customer's own NetBird dashboard
(Settings > Service Users / Personal Access Tokens) and paste it here.
"""
token: str = Field(..., min_length=10, max_length=500)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Users # Users
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
+13
View File
@@ -1260,6 +1260,19 @@
<p class="text-muted mb-0" data-i18n="monitoring.clickCheckUpdates">Click "Check for Updates" to compare local images with Docker Hub.</p> <p class="text-muted mb-0" data-i18n="monitoring.clickCheckUpdates">Click "Check for Updates" to compare local images with Docker Hub.</p>
</div> </div>
</div> </div>
<!-- Update Run Log -->
<div class="card shadow-sm mt-4">
<div class="card-header d-flex justify-content-between align-items-center">
<span><i class="bi bi-clock-history me-2"></i><span data-i18n="monitoring.updateLogTitle">Update Log</span></span>
<button class="btn btn-outline-secondary btn-sm" onclick="loadUpdateLog()" id="btn-refresh-update-log">
<i class="bi bi-arrow-clockwise me-1"></i><span data-i18n="common.refresh">Refresh</span>
</button>
</div>
<div class="card-body" id="update-log-body">
<p class="text-muted mb-0" data-i18n="common.loading">Loading...</p>
</div>
</div>
</div> </div>
</div> </div>
</div> </div>
+110 -2
View File
@@ -605,7 +605,17 @@ async function loadCustomerNetbirdUpdates(id, hasToken) {
try { try {
const data = await api('GET', `/customers/${id}/netbird-updates`); const data = await api('GET', `/customers/${id}/netbird-updates`);
const isCustom = data.version && data.version !== 'disabled' && data.version !== 'latest'; const isCustom = data.version && data.version !== 'disabled' && data.version !== 'latest';
let tokenInfoHtml = '';
if (data.token_renewed_at) {
const renewedDate = new Date(data.token_renewed_at);
const expiresDate = new Date(renewedDate);
expiresDate.setDate(expiresDate.getDate() + 365);
tokenInfoHtml = `<div class="small text-muted mb-2">${t('customer.nbuTokenRenewed', { date: renewedDate.toLocaleDateString() })} &middot; ${t('customer.nbuTokenExpires', { date: expiresDate.toLocaleDateString() })}</div>`;
}
container.innerHTML = ` container.innerHTML = `
${tokenInfoHtml}
<div class="row g-2 align-items-end"> <div class="row g-2 align-items-end">
<div class="col-auto"> <div class="col-auto">
<label class="form-label small mb-1">${t('customer.nbuVersion')}</label> <label class="form-label small mb-1">${t('customer.nbuVersion')}</label>
@@ -625,12 +635,27 @@ async function loadCustomerNetbirdUpdates(id, hasToken) {
<button class="btn btn-outline-secondary btn-sm" onclick="syncCustomerNetbirdFromMaster(${id})">${t('customer.nbuSyncMaster')}</button> <button class="btn btn-outline-secondary btn-sm" onclick="syncCustomerNetbirdFromMaster(${id})">${t('customer.nbuSyncMaster')}</button>
</div> </div>
</div> </div>
<div id="nbu-result" class="small mt-2"></div>`; <div id="nbu-result" class="small mt-2"></div>
<div class="mt-2">
<a href="#" class="small" onclick="event.preventDefault(); showChangeNetbirdToken(${id})">${t('customer.nbuChangeToken')}</a>
</div>
<div id="nbu-change-token-area" class="mt-2 d-none"></div>`;
} catch (err) { } catch (err) {
container.innerHTML = `<div class="alert alert-warning py-2 small mb-0">${esc(err.message)}</div>`; container.innerHTML = `<div class="alert alert-warning py-2 small mb-0">${esc(err.message)}</div>`;
} }
} }
function showChangeNetbirdToken(id) {
const area = document.getElementById('nbu-change-token-area');
area.classList.remove('d-none');
area.innerHTML = `
<div class="input-group input-group-sm">
<input type="text" class="form-control" id="nbu-token-input" placeholder="${t('customer.nbuTokenPlaceholder')}">
<button class="btn btn-outline-primary" onclick="saveCustomerNetbirdToken(${id})">${t('customer.nbuSaveToken')}</button>
</div>
<div id="nbu-token-result" class="small mt-1"></div>`;
}
async function saveCustomerNetbirdToken(id) { async function saveCustomerNetbirdToken(id) {
const input = document.getElementById('nbu-token-input'); const input = document.getElementById('nbu-token-input');
const resultEl = document.getElementById('nbu-token-result'); const resultEl = document.getElementById('nbu-token-result');
@@ -1918,7 +1943,90 @@ function togglePasswordVisibility(inputId) {
// Monitoring // Monitoring
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
async function loadMonitoring() { async function loadMonitoring() {
await Promise.all([loadResources(), loadAllCustomerStatuses()]); await Promise.all([loadResources(), loadAllCustomerStatuses(), loadUpdateLog()]);
}
const _updateLogStatusBadge = {
success: 'bg-success',
no_update: 'bg-secondary',
partial: 'bg-warning text-dark',
failed: 'bg-danger',
running: 'bg-info',
};
async function loadUpdateLog() {
const body = document.getElementById('update-log-body');
body.innerHTML = `<span class="spinner-border spinner-border-sm me-2"></span>${t('common.loading')}`;
try {
const logs = await api('GET', '/monitoring/update-logs?limit=30');
if (logs.length === 0) {
body.innerHTML = `<p class="text-muted mb-0">${t('monitoring.noUpdateLogs')}</p>`;
return;
}
const rows = logs.map((l, i) => {
const badgeClass = _updateLogStatusBadge[l.status] || 'bg-secondary';
const typeLabel = l.run_type === 'scheduled' ? t('monitoring.logTypeScheduled') : t('monitoring.logTypeManual');
const pullSummary = l.pull_attempted
? (l.pull_success ? t('monitoring.logPullOk') : t('monitoring.logPullFailed'))
: t('monitoring.logPullSkipped');
const applySummary = l.apply_attempted
? t('monitoring.logApplySummary', { updated: l.customers_updated ?? 0, total: l.customers_total ?? 0 })
: t('monitoring.logApplySkipped');
const detailId = `update-log-detail-${i}`;
const hasDetails = (l.pull_details && Object.keys(l.pull_details).length > 0) ||
(l.customer_details && l.customer_details.length > 0) || l.error;
const detailsHtml = hasDetails ? `
<div class="collapse mt-2" id="${detailId}">
${l.error ? `<div class="alert alert-danger py-2 small mb-2">${esc(l.error)}</div>` : ''}
${l.pull_details ? `
<table class="table table-sm mb-2">
<thead><tr><th>${t('monitoring.thImage')}</th><th>${t('monitoring.thStatus')}</th><th></th></tr></thead>
<tbody>${Object.entries(l.pull_details).map(([img, r]) => `<tr>
<td><code class="small">${esc(img)}</code></td>
<td>${r.success ? `<span class="badge bg-success">OK</span>` : `<span class="badge bg-danger">Error</span>`}</td>
<td class="small text-muted">${esc(r.error || '')}</td>
</tr>`).join('')}</tbody>
</table>` : ''}
${l.customer_details && l.customer_details.length > 0 ? `
<table class="table table-sm mb-0">
<thead><tr><th>${t('monitoring.thName')}</th><th>${t('monitoring.thStatus')}</th><th></th></tr></thead>
<tbody>${l.customer_details.map(r => `<tr>
<td>${esc(r.customer_name)}</td>
<td>${r.success ? `<span class="badge bg-success">OK</span>` : `<span class="badge bg-danger">Error</span>`}</td>
<td class="small text-muted">${esc(r.error || '')}</td>
</tr>`).join('')}</tbody>
</table>` : ''}
</div>` : '';
return `<tr>
<td class="small text-nowrap">${formatDate(l.started_at)}</td>
<td><span class="badge bg-light text-dark border">${typeLabel}</span></td>
<td><span class="badge ${badgeClass}">${t('monitoring.logStatus_' + l.status)}</span></td>
<td class="small">${pullSummary}</td>
<td class="small">${applySummary}</td>
<td class="small text-muted">${esc(l.trigger || '')}</td>
<td>${hasDetails ? `<button class="btn btn-sm btn-outline-secondary" data-bs-toggle="collapse" data-bs-target="#${detailId}"><i class="bi bi-chevron-down"></i></button>` : ''}</td>
</tr>${detailsHtml ? `<tr class="d-none"></tr><tr><td colspan="7" class="p-0 border-0">${detailsHtml}</td></tr>` : ''}`;
}).join('');
body.innerHTML = `
<div class="table-responsive">
<table class="table table-sm mb-0">
<thead class="table-light">
<tr>
<th>${t('monitoring.logTime')}</th>
<th>${t('monitoring.logType')}</th>
<th>${t('monitoring.thStatus')}</th>
<th>${t('monitoring.logPull')}</th>
<th>${t('monitoring.logApply')}</th>
<th>${t('monitoring.logTrigger')}</th>
<th></th>
</tr>
</thead>
<tbody>${rows}</tbody>
</table>
</div>`;
} catch (err) {
body.innerHTML = `<div class="alert alert-danger">${esc(err.message)}</div>`;
}
} }
async function loadResources() { async function loadResources() {
+23 -1
View File
@@ -97,6 +97,9 @@
"nbuTokenPlaceholder": "Personal Access Token einfügen…", "nbuTokenPlaceholder": "Personal Access Token einfügen…",
"nbuSaveToken": "Prüfen & Speichern", "nbuSaveToken": "Prüfen & Speichern",
"nbuTokenSaved": "Token gespeichert.", "nbuTokenSaved": "Token gespeichert.",
"nbuChangeToken": "Token ändern",
"nbuTokenRenewed": "Token erneuert am {date}",
"nbuTokenExpires": "gültig bis {date}",
"nbuVersion": "Client-Version", "nbuVersion": "Client-Version",
"nbuDisabled": "Deaktiviert", "nbuDisabled": "Deaktiviert",
"nbuLatest": "Neueste Version", "nbuLatest": "Neueste Version",
@@ -451,6 +454,25 @@
"autoUpdateApplyHint": "Wenn aktiviert, werden nach einer gefundenen Aktualisierung automatisch alle Kunden-Container neu erstellt (kurzer Neustart der Dienste). Wenn deaktiviert, werden nur die Images geladen — die Aktualisierung der Kunden erfolgt weiterhin manuell.", "autoUpdateApplyHint": "Wenn aktiviert, werden nach einer gefundenen Aktualisierung automatisch alle Kunden-Container neu erstellt (kurzer Neustart der Dienste). Wenn deaktiviert, werden nur die Images geladen — die Aktualisierung der Kunden erfolgt weiterhin manuell.",
"autoUpdateLastRun": "Letzte automatische Prüfung", "autoUpdateLastRun": "Letzte automatische Prüfung",
"autoUpdateNever": "Noch nie ausgeführt", "autoUpdateNever": "Noch nie ausgeführt",
"saveAutoUpdateSettings": "Automatisierung speichern" "saveAutoUpdateSettings": "Automatisierung speichern",
"updateLogTitle": "Update-Verlauf",
"noUpdateLogs": "Noch keine Update-Läufe protokolliert.",
"logTypeScheduled": "Automatisch",
"logTypeManual": "Manuell",
"logTime": "Zeitpunkt",
"logType": "Typ",
"logPull": "Image-Download",
"logApply": "Kunden-Update",
"logTrigger": "Ausgelöst von",
"logPullOk": "Erfolgreich",
"logPullFailed": "Fehlgeschlagen",
"logPullSkipped": "Nicht nötig",
"logApplySummary": "{updated}/{total} Kunden",
"logApplySkipped": "Nicht ausgeführt",
"logStatus_success": "Erfolgreich",
"logStatus_no_update": "Kein Update nötig",
"logStatus_partial": "Teilweise",
"logStatus_failed": "Fehlgeschlagen",
"logStatus_running": "Läuft…"
} }
} }
+23 -1
View File
@@ -97,6 +97,9 @@
"nbuTokenPlaceholder": "Paste Personal Access Token…", "nbuTokenPlaceholder": "Paste Personal Access Token…",
"nbuSaveToken": "Verify & Save", "nbuSaveToken": "Verify & Save",
"nbuTokenSaved": "Token saved.", "nbuTokenSaved": "Token saved.",
"nbuChangeToken": "Change token",
"nbuTokenRenewed": "Token renewed on {date}",
"nbuTokenExpires": "valid until {date}",
"nbuVersion": "Client version", "nbuVersion": "Client version",
"nbuDisabled": "Disabled", "nbuDisabled": "Disabled",
"nbuLatest": "Latest version", "nbuLatest": "Latest version",
@@ -358,7 +361,26 @@
"autoUpdateApplyHint": "When enabled, all customer containers are automatically recreated after a new image is found (services briefly restart). When disabled, only the images are pulled — updating customers stays a manual step.", "autoUpdateApplyHint": "When enabled, all customer containers are automatically recreated after a new image is found (services briefly restart). When disabled, only the images are pulled — updating customers stays a manual step.",
"autoUpdateLastRun": "Last automatic check", "autoUpdateLastRun": "Last automatic check",
"autoUpdateNever": "Never run", "autoUpdateNever": "Never run",
"saveAutoUpdateSettings": "Save Automation" "saveAutoUpdateSettings": "Save Automation",
"updateLogTitle": "Update Log",
"noUpdateLogs": "No update runs logged yet.",
"logTypeScheduled": "Automatic",
"logTypeManual": "Manual",
"logTime": "Time",
"logType": "Type",
"logPull": "Image Pull",
"logApply": "Customer Update",
"logTrigger": "Triggered by",
"logPullOk": "Succeeded",
"logPullFailed": "Failed",
"logPullSkipped": "Not needed",
"logApplySummary": "{updated}/{total} customers",
"logApplySkipped": "Not run",
"logStatus_success": "Success",
"logStatus_no_update": "No update needed",
"logStatus_partial": "Partial",
"logStatus_failed": "Failed",
"logStatus_running": "Running…"
}, },
"userModal": { "userModal": {
"title": "New User", "title": "New User",