Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
75357b0633 | ||
|
|
556db5bfd4 | ||
|
|
d4b6efeea1 | ||
|
|
276f4214a7 | ||
|
|
a8ed87faa9 | ||
|
|
1bc768151a | ||
|
|
1e05ddfe69 | ||
|
|
e9afdb226c | ||
|
|
8a84e60a27 |
@@ -45,6 +45,7 @@ def init_db() -> None:
|
||||
Deployment,
|
||||
DeploymentLog,
|
||||
SystemConfig,
|
||||
UpdateRunLog,
|
||||
User,
|
||||
)
|
||||
|
||||
@@ -129,6 +130,7 @@ def _run_migrations() -> None:
|
||||
("system_config", "auto_update_last_run_at", "TEXT"),
|
||||
# NetBird client (peer) automatic-updates master default + per-customer PAT
|
||||
("deployments", "netbird_api_token_encrypted", "TEXT"),
|
||||
("deployments", "netbird_api_token_renewed_at", "TEXT"),
|
||||
("system_config", "netbird_client_auto_update_version", "TEXT DEFAULT 'disabled'"),
|
||||
("system_config", "netbird_client_auto_update_always", "BOOLEAN DEFAULT 0"),
|
||||
]
|
||||
|
||||
+1
-1
@@ -34,7 +34,7 @@ logger = logging.getLogger(__name__)
|
||||
app = FastAPI(
|
||||
title="NetBird MSP Appliance",
|
||||
description="Multi-tenant NetBird management platform for MSPs",
|
||||
version="1.4.0",
|
||||
version="1.5.1",
|
||||
docs_url="/api/docs",
|
||||
redoc_url="/api/redoc",
|
||||
openapi_url="/api/openapi.json",
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""SQLAlchemy ORM models for NetBird MSP Appliance."""
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
@@ -89,6 +90,7 @@ class Deployment(Base):
|
||||
netbird_admin_email: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||
netbird_admin_password: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||
netbird_api_token_encrypted: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||
netbird_api_token_renewed_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
|
||||
deployment_status: Mapped[str] = mapped_column(
|
||||
String(20), default="pending", nullable=False
|
||||
)
|
||||
@@ -118,6 +120,9 @@ class Deployment(Base):
|
||||
"setup_url": self.setup_url,
|
||||
"has_credentials": bool(self.netbird_admin_email and self.netbird_admin_password),
|
||||
"has_netbird_api_token": bool(self.netbird_api_token_encrypted),
|
||||
"netbird_api_token_renewed_at": (
|
||||
self.netbird_api_token_renewed_at.isoformat() if self.netbird_api_token_renewed_at else None
|
||||
),
|
||||
"deployment_status": self.deployment_status,
|
||||
"deployed_at": self.deployed_at.isoformat() if self.deployed_at else None,
|
||||
"last_health_check": (
|
||||
@@ -317,6 +322,56 @@ class DeploymentLog(Base):
|
||||
}
|
||||
|
||||
|
||||
class UpdateRunLog(Base):
|
||||
"""Audit trail for NetBird Docker image update runs (pull + container recreate).
|
||||
|
||||
Written for both scheduled (automatic) and manual (button-triggered)
|
||||
runs, so the actual outcome — which image pulls failed, which customer
|
||||
container recreates failed — is visible in the UI instead of only in
|
||||
container logs.
|
||||
"""
|
||||
|
||||
__tablename__ = "update_run_logs"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
run_type: Mapped[str] = mapped_column(String(20), nullable=False) # "scheduled" | "manual"
|
||||
trigger: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) # username, or "scheduler"
|
||||
started_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
|
||||
finished_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(20), default="running"
|
||||
) # running | no_update | success | partial | failed
|
||||
any_update_available: Mapped[Optional[bool]] = mapped_column(Boolean, nullable=True)
|
||||
pull_attempted: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
pull_success: Mapped[Optional[bool]] = mapped_column(Boolean, nullable=True)
|
||||
pull_details: Mapped[Optional[str]] = mapped_column(Text, nullable=True) # JSON: {image: {success, error}}
|
||||
apply_attempted: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
customers_updated: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
|
||||
customers_total: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
|
||||
customer_details: Mapped[Optional[str]] = mapped_column(Text, nullable=True) # JSON list
|
||||
error: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Serialize to dict, parsing the embedded JSON blobs."""
|
||||
return {
|
||||
"id": self.id,
|
||||
"run_type": self.run_type,
|
||||
"trigger": self.trigger,
|
||||
"started_at": self.started_at.isoformat() if self.started_at else None,
|
||||
"finished_at": self.finished_at.isoformat() if self.finished_at else None,
|
||||
"status": self.status,
|
||||
"any_update_available": self.any_update_available,
|
||||
"pull_attempted": self.pull_attempted,
|
||||
"pull_success": self.pull_success,
|
||||
"pull_details": json.loads(self.pull_details) if self.pull_details else None,
|
||||
"apply_attempted": self.apply_attempted,
|
||||
"customers_updated": self.customers_updated,
|
||||
"customers_total": self.customers_total,
|
||||
"customer_details": json.loads(self.customer_details) if self.customer_details else None,
|
||||
"error": self.error,
|
||||
}
|
||||
|
||||
|
||||
class User(Base):
|
||||
"""Admin user model."""
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Deployment management API — start, stop, restart, logs, health for customers."""
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query, status
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -8,7 +9,7 @@ from sqlalchemy.orm import Session
|
||||
from app.database import SessionLocal, get_db
|
||||
from app.dependencies import get_current_user
|
||||
from app.models import Customer, Deployment, SystemConfig, User
|
||||
from app.services import docker_service, image_service, netbird_client_update_service, netbird_service
|
||||
from app.services import docker_service, image_service, netbird_client_update_service, netbird_service, update_log_service
|
||||
from app.utils.security import decrypt_value, encrypt_value
|
||||
from app.utils.validators import NetbirdApiTokenPayload, NetbirdClientAutoUpdatePayload
|
||||
|
||||
@@ -245,9 +246,18 @@ async def update_customer_images(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="System not configured."
|
||||
)
|
||||
|
||||
started_at = datetime.utcnow()
|
||||
instance_dir = f"{config.data_dir}/{customer.subdomain}"
|
||||
result = await image_service.update_customer_containers(instance_dir, deployment.container_prefix)
|
||||
|
||||
update_log_service.record_run(
|
||||
db, run_type="manual", trigger=current_user.username, started_at=started_at,
|
||||
apply_attempted=True,
|
||||
customer_results=[{
|
||||
"customer_name": customer.name, "success": result["success"], "error": result.get("error"),
|
||||
}],
|
||||
)
|
||||
|
||||
if not result["success"]:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
@@ -287,6 +297,10 @@ async def get_customer_netbird_updates(
|
||||
settings = result["settings"]
|
||||
return {
|
||||
"has_token": True,
|
||||
"token_renewed_at": (
|
||||
deployment.netbird_api_token_renewed_at.isoformat()
|
||||
if deployment.netbird_api_token_renewed_at else None
|
||||
),
|
||||
"version": settings.get("auto_update_version", "disabled"),
|
||||
"always": bool(settings.get("auto_update_always", False)),
|
||||
}
|
||||
@@ -362,6 +376,7 @@ async def set_customer_netbird_api_token(
|
||||
)
|
||||
|
||||
deployment.netbird_api_token_encrypted = encrypt_value(payload.token)
|
||||
deployment.netbird_api_token_renewed_at = datetime.utcnow()
|
||||
db.commit()
|
||||
logger.info("NetBird API token registered for customer %d by %s.", customer_id, current_user.username)
|
||||
return {"ok": True}
|
||||
|
||||
@@ -4,6 +4,7 @@ import asyncio
|
||||
import logging
|
||||
import platform
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import psutil
|
||||
@@ -12,8 +13,8 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import SessionLocal, get_db
|
||||
from app.dependencies import get_current_user
|
||||
from app.models import Customer, Deployment, SystemConfig, User
|
||||
from app.services import docker_service, image_service, netbird_client_update_service
|
||||
from app.models import Customer, Deployment, SystemConfig, UpdateRunLog, User
|
||||
from app.services import docker_service, image_service, netbird_client_update_service, update_log_service
|
||||
from app.utils.security import decrypt_value
|
||||
from app.utils.validators import NetbirdClientAutoUpdatePayload
|
||||
|
||||
@@ -194,14 +195,25 @@ async def pull_all_netbird_images(
|
||||
config.netbird_dashboard_image,
|
||||
]
|
||||
|
||||
username = current_user.username
|
||||
|
||||
async def _pull_bg() -> None:
|
||||
started_at = datetime.utcnow()
|
||||
bg_db = SessionLocal()
|
||||
try:
|
||||
cfg = bg_db.query(SystemConfig).filter(SystemConfig.id == 1).first()
|
||||
if cfg:
|
||||
await image_service.pull_all_images(cfg)
|
||||
except Exception:
|
||||
if not cfg:
|
||||
return
|
||||
pull_result = await image_service.pull_all_images(cfg)
|
||||
update_log_service.record_run(
|
||||
bg_db, run_type="manual", trigger=username, started_at=started_at,
|
||||
any_update_available=True, pull_attempted=True, pull_results=pull_result["results"],
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.exception("Background image pull failed")
|
||||
update_log_service.record_run(
|
||||
bg_db, run_type="manual", trigger=username, started_at=started_at, error=str(exc),
|
||||
)
|
||||
finally:
|
||||
bg_db.close()
|
||||
|
||||
@@ -285,6 +297,28 @@ async def apply_netbird_client_updates_to_all(
|
||||
}
|
||||
|
||||
|
||||
@router.get("/update-logs")
|
||||
async def get_update_logs(
|
||||
limit: int = 30,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return the most recent NetBird image update runs (scheduled and manual).
|
||||
|
||||
Shows exactly what happened on each run: whether an image pull was
|
||||
attempted and its per-image result, and whether customer containers
|
||||
were recreated and which ones failed — so this doesn't have to be
|
||||
reconstructed from container logs.
|
||||
"""
|
||||
logs = (
|
||||
db.query(UpdateRunLog)
|
||||
.order_by(UpdateRunLog.started_at.desc())
|
||||
.limit(min(limit, 100))
|
||||
.all()
|
||||
)
|
||||
return [log.to_dict() for log in logs]
|
||||
|
||||
|
||||
@router.post("/customers/update-all")
|
||||
async def update_all_customers(
|
||||
current_user: User = Depends(get_current_user),
|
||||
@@ -299,6 +333,7 @@ async def update_all_customers(
|
||||
if current_user.role != "admin":
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin only.")
|
||||
|
||||
started_at = datetime.utcnow()
|
||||
config = db.query(SystemConfig).filter(SystemConfig.id == 1).first()
|
||||
if not config:
|
||||
raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="System not configured.")
|
||||
@@ -317,6 +352,10 @@ async def update_all_customers(
|
||||
})
|
||||
|
||||
if not to_update:
|
||||
update_log_service.record_run(
|
||||
db, run_type="manual", trigger=current_user.username,
|
||||
started_at=started_at, apply_attempted=True, customer_results=[],
|
||||
)
|
||||
return {"message": "All customers are already up to date.", "updated": 0, "results": []}
|
||||
|
||||
# Update customers sequentially — one at a time. A failure for one
|
||||
@@ -343,6 +382,10 @@ async def update_all_customers(
|
||||
})
|
||||
|
||||
success_count = sum(1 for r in update_results if r["success"])
|
||||
update_log_service.record_run(
|
||||
db, run_type="manual", trigger=current_user.username,
|
||||
started_at=started_at, apply_attempted=True, customer_results=update_results,
|
||||
)
|
||||
return {
|
||||
"message": f"Updated {success_count} of {len(update_results)} customer(s).",
|
||||
"updated": success_count,
|
||||
|
||||
@@ -52,6 +52,48 @@ async def get_current_settings(container_prefix: str, token: str) -> dict[str, A
|
||||
return {"ok": False, "error": str(exc)}
|
||||
|
||||
|
||||
async def renew_token(container_prefix: str, token: str) -> dict[str, Any]:
|
||||
"""Mint a fresh 365-day Personal Access Token using the current one.
|
||||
|
||||
The old token is left in place (it naturally expires and NetBird gives no
|
||||
reliable way to identify "our" token among a user's other PATs by content
|
||||
alone, only by name — which isn't safe to assume is unique). One
|
||||
harmless unused token lingering until its own expiry is an acceptable
|
||||
trade-off for not risking deleting a token that turns out to be in use.
|
||||
"""
|
||||
base_url = _base_url(container_prefix)
|
||||
headers = {"Authorization": f"Token {token}"}
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
|
||||
resp = await client.get(f"{base_url}/api/users", headers=headers)
|
||||
if resp.status_code != 200:
|
||||
return {"ok": False, "error": f"GET /api/users -> HTTP {resp.status_code}"}
|
||||
users = resp.json()
|
||||
me = next((u for u in users if u.get("is_current")), None)
|
||||
if not me:
|
||||
return {"ok": False, "error": "Could not identify current user from /api/users."}
|
||||
|
||||
create_resp = await client.post(
|
||||
f"{base_url}/api/users/{me['id']}/tokens",
|
||||
headers={**headers, "Content-Type": "application/json"},
|
||||
content=json.dumps({"name": "MSP Central Management", "expires_in": 365}),
|
||||
)
|
||||
if create_resp.status_code not in (200, 201):
|
||||
return {"ok": False, "error": f"POST tokens -> HTTP {create_resp.status_code}: {create_resp.text[:200]}"}
|
||||
new_token = create_resp.json().get("plain_token")
|
||||
if not new_token:
|
||||
return {"ok": False, "error": "Token creation response had no plain_token."}
|
||||
|
||||
verify_resp = await client.get(f"{base_url}/api/accounts", headers={"Authorization": f"Token {new_token}"})
|
||||
if verify_resp.status_code != 200:
|
||||
return {"ok": False, "error": "New token failed verification."}
|
||||
|
||||
return {"ok": True, "token": new_token}
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to renew NetBird API token for %s: %s", container_prefix, exc)
|
||||
return {"ok": False, "error": str(exc)}
|
||||
|
||||
|
||||
async def push_auto_update_settings(
|
||||
container_prefix: str, token: str, version: str, always: bool
|
||||
) -> dict[str, Any]:
|
||||
|
||||
@@ -348,9 +348,9 @@ async def deploy_customer(db: Session, customer_id: int) -> dict[str, Any]:
|
||||
deployment.setup_url = setup_url
|
||||
deployment.netbird_admin_email = encrypt_value(admin_email) if setup_ok else deployment.netbird_admin_email
|
||||
deployment.netbird_admin_password = encrypt_value(admin_password) if setup_ok else deployment.netbird_admin_password
|
||||
deployment.netbird_api_token_encrypted = (
|
||||
encrypt_value(netbird_api_token) if netbird_api_token else deployment.netbird_api_token_encrypted
|
||||
)
|
||||
if netbird_api_token:
|
||||
deployment.netbird_api_token_encrypted = encrypt_value(netbird_api_token)
|
||||
deployment.netbird_api_token_renewed_at = datetime.utcnow()
|
||||
deployment.deployment_status = "running"
|
||||
deployment.deployed_at = datetime.utcnow()
|
||||
else:
|
||||
@@ -366,6 +366,7 @@ async def deploy_customer(db: Session, customer_id: int) -> dict[str, Any]:
|
||||
netbird_admin_email=encrypt_value(admin_email) if setup_ok else None,
|
||||
netbird_admin_password=encrypt_value(admin_password) if setup_ok else None,
|
||||
netbird_api_token_encrypted=encrypt_value(netbird_api_token) if netbird_api_token else None,
|
||||
netbird_api_token_renewed_at=datetime.utcnow() if netbird_api_token else None,
|
||||
deployment_status="running",
|
||||
deployed_at=datetime.utcnow(),
|
||||
)
|
||||
|
||||
@@ -8,17 +8,23 @@ SystemConfig.auto_update_check_enabled / auto_update_check_time.
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from app.database import SessionLocal
|
||||
from app.models import Deployment, SystemConfig
|
||||
from app.services import image_service
|
||||
from app.services import image_service, netbird_client_update_service, update_log_service
|
||||
from app.utils.security import decrypt_value, encrypt_value
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_POLL_INTERVAL_SECONDS = 60
|
||||
_task: asyncio.Task | None = None
|
||||
|
||||
# NetBird PATs we mint are issued for 365 days; renew well before that so a
|
||||
# missed tick or a slow rollout never risks the token actually expiring.
|
||||
_TOKEN_RENEW_AFTER_DAYS = 300
|
||||
_TOKEN_RENEW_CHECK_HOUR = 4 # run once per day, distinct from the image-check hour
|
||||
|
||||
|
||||
def start() -> None:
|
||||
"""Start the background polling task. Safe to call once at app startup."""
|
||||
@@ -36,15 +42,64 @@ def stop() -> None:
|
||||
_task = None
|
||||
|
||||
|
||||
_last_token_renewal_date = None
|
||||
|
||||
|
||||
async def _poll_loop() -> None:
|
||||
while True:
|
||||
try:
|
||||
await _tick()
|
||||
except Exception:
|
||||
logger.exception("Scheduler tick failed")
|
||||
try:
|
||||
await _token_renewal_tick()
|
||||
except Exception:
|
||||
logger.exception("Token renewal tick failed")
|
||||
await asyncio.sleep(_POLL_INTERVAL_SECONDS)
|
||||
|
||||
|
||||
async def _token_renewal_tick() -> None:
|
||||
"""Once a day, renew any NetBird client-update API token nearing its
|
||||
365-day expiry — keeps central update control working indefinitely
|
||||
without anyone needing to notice or act.
|
||||
"""
|
||||
global _last_token_renewal_date
|
||||
now = datetime.now()
|
||||
if now.hour != _TOKEN_RENEW_CHECK_HOUR:
|
||||
return
|
||||
if _last_token_renewal_date == now.date():
|
||||
return
|
||||
_last_token_renewal_date = now.date()
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
cutoff = now - timedelta(days=_TOKEN_RENEW_AFTER_DAYS)
|
||||
deployments = (
|
||||
db.query(Deployment)
|
||||
.filter(Deployment.netbird_api_token_encrypted.isnot(None))
|
||||
.all()
|
||||
)
|
||||
due = [
|
||||
d for d in deployments
|
||||
if d.netbird_api_token_renewed_at is None or d.netbird_api_token_renewed_at < cutoff
|
||||
]
|
||||
if not due:
|
||||
return
|
||||
logger.info("Renewing NetBird API token for %d customer(s)...", len(due))
|
||||
for d in due:
|
||||
token = decrypt_value(d.netbird_api_token_encrypted)
|
||||
result = await netbird_client_update_service.renew_token(d.container_prefix, token)
|
||||
if result["ok"]:
|
||||
d.netbird_api_token_encrypted = encrypt_value(result["token"])
|
||||
d.netbird_api_token_renewed_at = now
|
||||
db.commit()
|
||||
logger.info("Renewed NetBird API token for %s.", d.container_prefix)
|
||||
else:
|
||||
logger.warning("Failed to renew NetBird API token for %s: %s", d.container_prefix, result.get("error"))
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
async def _tick() -> None:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
@@ -70,29 +125,49 @@ async def _tick() -> None:
|
||||
logger.info(
|
||||
"Running scheduled NetBird image update check (auto-apply=%s)...", apply_enabled
|
||||
)
|
||||
await _run_check_and_optionally_apply(config, apply_enabled)
|
||||
await _run_check_and_optionally_apply(config, apply_enabled, started_at=now)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
async def _run_check_and_optionally_apply(config: SystemConfig, apply_enabled: bool) -> None:
|
||||
hub_status = await image_service.check_all_images(config)
|
||||
if not hub_status["any_update_available"]:
|
||||
logger.info("Scheduled check: all NetBird images already up to date.")
|
||||
return
|
||||
async def _run_check_and_optionally_apply(config: SystemConfig, apply_enabled: bool, started_at: datetime) -> None:
|
||||
"""Check Docker Hub, pull if needed, then (if enabled) recreate any
|
||||
customer container that isn't running the locally-pulled image yet.
|
||||
|
||||
logger.info("Scheduled check: new NetBird image(s) available — pulling.")
|
||||
pull_result = await image_service.pull_all_images(config)
|
||||
if not pull_result["all_success"]:
|
||||
logger.error("Scheduled image pull had failures: %s", pull_result["results"])
|
||||
|
||||
if not apply_enabled:
|
||||
logger.info("Auto-apply disabled — images pulled, customer containers left untouched.")
|
||||
return
|
||||
|
||||
db = SessionLocal()
|
||||
The "recreate" step is intentionally NOT gated on a Hub update having
|
||||
just been found: a customer can already be behind the *locally pulled*
|
||||
image from an earlier check/pull that was never applied (e.g. auto-apply
|
||||
was off at the time, or a previous run failed) — that backlog must keep
|
||||
being cleared on every run, not just on runs that also found something
|
||||
new on Hub, otherwise it silently never happens.
|
||||
"""
|
||||
log_db = SessionLocal()
|
||||
try:
|
||||
deployments = db.query(Deployment).all()
|
||||
hub_status = await image_service.check_all_images(config)
|
||||
any_update = hub_status["any_update_available"]
|
||||
|
||||
pull_attempted = False
|
||||
pull_results = None
|
||||
if any_update:
|
||||
logger.info("Scheduled check: new NetBird image(s) available — pulling.")
|
||||
pull_attempted = True
|
||||
pull_result = await image_service.pull_all_images(config)
|
||||
pull_results = pull_result["results"]
|
||||
if not pull_result["all_success"]:
|
||||
logger.error("Scheduled image pull had failures: %s", pull_results)
|
||||
else:
|
||||
logger.info("Scheduled check: all NetBird images already up to date on Docker Hub.")
|
||||
|
||||
if not apply_enabled:
|
||||
logger.info("Auto-apply disabled — customer containers left untouched.")
|
||||
update_log_service.record_run(
|
||||
log_db, run_type="scheduled", trigger="scheduler",
|
||||
started_at=started_at, any_update_available=any_update,
|
||||
pull_attempted=pull_attempted, pull_results=pull_results,
|
||||
)
|
||||
return
|
||||
|
||||
deployments = log_db.query(Deployment).all()
|
||||
to_update = []
|
||||
for dep in deployments:
|
||||
cs = image_service.get_customer_container_image_status(dep.container_prefix, config)
|
||||
@@ -104,16 +179,32 @@ async def _run_check_and_optionally_apply(config: SystemConfig, apply_enabled: b
|
||||
"customer_name": customer.name,
|
||||
})
|
||||
logger.info("Scheduled auto-apply: updating %d customer(s)...", len(to_update))
|
||||
customer_results = []
|
||||
for entry in to_update:
|
||||
try:
|
||||
res = await image_service.update_customer_containers(
|
||||
entry["instance_dir"], entry["project_name"]
|
||||
)
|
||||
logger.info(
|
||||
"Scheduled update for %s: %s",
|
||||
entry["customer_name"], "OK" if res["success"] else res.get("error"),
|
||||
)
|
||||
except Exception:
|
||||
ok = res["success"]
|
||||
err = res.get("error")
|
||||
except Exception as exc:
|
||||
logger.exception("Scheduled update failed for %s", entry["customer_name"])
|
||||
ok = False
|
||||
err = str(exc)
|
||||
logger.info("Scheduled update for %s: %s", entry["customer_name"], "OK" if ok else err)
|
||||
customer_results.append({"customer_name": entry["customer_name"], "success": ok, "error": err})
|
||||
|
||||
update_log_service.record_run(
|
||||
log_db, run_type="scheduled", trigger="scheduler",
|
||||
started_at=started_at, any_update_available=any_update,
|
||||
pull_attempted=pull_attempted, pull_results=pull_results,
|
||||
apply_attempted=True, customer_results=customer_results,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.exception("Scheduled update run failed")
|
||||
update_log_service.record_run(
|
||||
log_db, run_type="scheduled", trigger="scheduler",
|
||||
started_at=started_at, error=str(exc),
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
log_db.close()
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Persists the outcome of every NetBird Docker image update run.
|
||||
|
||||
Both the automatic scheduler and the manual "Check/Pull/Update" buttons in
|
||||
the UI write here, so it's visible afterwards which run pulled which image
|
||||
successfully, and which customer container recreate failed — instead of
|
||||
that information only existing in container logs.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Any, Optional
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import UpdateRunLog
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def record_run(
|
||||
db: Session,
|
||||
*,
|
||||
run_type: str,
|
||||
trigger: Optional[str],
|
||||
started_at: datetime,
|
||||
any_update_available: Optional[bool] = None,
|
||||
pull_attempted: bool = False,
|
||||
pull_results: Optional[dict[str, Any]] = None,
|
||||
apply_attempted: bool = False,
|
||||
customer_results: Optional[list[dict[str, Any]]] = None,
|
||||
error: Optional[str] = None,
|
||||
) -> UpdateRunLog:
|
||||
"""Write one completed update-run entry.
|
||||
|
||||
Args:
|
||||
run_type: "scheduled" or "manual".
|
||||
trigger: username for manual runs, "scheduler" for automatic ones.
|
||||
pull_results: {image: {"success": bool, "error": str|None}} as returned
|
||||
by image_service.pull_all_images()["results"].
|
||||
customer_results: list of {"customer_name", "success", "error"} as
|
||||
returned by the update-all / update-customer flows.
|
||||
"""
|
||||
pull_success = None
|
||||
if pull_results:
|
||||
pull_success = all(r.get("success") for r in pull_results.values())
|
||||
|
||||
customers_total = len(customer_results) if customer_results is not None else None
|
||||
customers_updated = (
|
||||
sum(1 for r in customer_results if r.get("success")) if customer_results is not None else None
|
||||
)
|
||||
|
||||
if error:
|
||||
status = "failed"
|
||||
elif pull_attempted and pull_success is False:
|
||||
status = "failed"
|
||||
elif apply_attempted and customers_total and customers_updated != customers_total:
|
||||
status = "partial"
|
||||
elif not pull_attempted and not customers_total:
|
||||
# Nothing to pull (Hub already matched local) and no customer
|
||||
# container was behind the locally pulled image either.
|
||||
status = "no_update"
|
||||
else:
|
||||
status = "success"
|
||||
|
||||
log = UpdateRunLog(
|
||||
run_type=run_type,
|
||||
trigger=trigger,
|
||||
started_at=started_at,
|
||||
finished_at=datetime.utcnow(),
|
||||
status=status,
|
||||
any_update_available=any_update_available,
|
||||
pull_attempted=pull_attempted,
|
||||
pull_success=pull_success,
|
||||
pull_details=json.dumps(pull_results) if pull_results is not None else None,
|
||||
apply_attempted=apply_attempted,
|
||||
customers_updated=customers_updated,
|
||||
customers_total=customers_total,
|
||||
customer_details=json.dumps(customer_results) if customer_results is not None else None,
|
||||
error=error,
|
||||
)
|
||||
db.add(log)
|
||||
db.commit()
|
||||
return log
|
||||
+21
-21
@@ -177,27 +177,6 @@ class SystemConfigUpdate(BaseModel):
|
||||
raise ValueError("auto_update_check_time must be in HH:MM 24h format")
|
||||
return v
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# NetBird client (peer) automatic updates
|
||||
# ---------------------------------------------------------------------------
|
||||
class NetbirdClientAutoUpdatePayload(BaseModel):
|
||||
"""Push a client auto-update version/mode to one or all customers."""
|
||||
|
||||
version: str = Field(..., max_length=50, description="'latest', 'disabled', or a version e.g. '0.61.0'")
|
||||
always: bool = False
|
||||
|
||||
|
||||
class NetbirdApiTokenPayload(BaseModel):
|
||||
"""Manually register a NetBird Personal Access Token for a customer.
|
||||
|
||||
Needed for customers deployed before automatic PAT capture existed —
|
||||
create a PAT once in that customer's own NetBird dashboard
|
||||
(Settings > Service Users / Personal Access Tokens) and paste it here.
|
||||
"""
|
||||
|
||||
token: str = Field(..., min_length=10, max_length=500)
|
||||
|
||||
@field_validator("ssl_mode")
|
||||
@classmethod
|
||||
def validate_ssl_mode(cls, v: Optional[str]) -> Optional[str]:
|
||||
@@ -242,6 +221,27 @@ class NetbirdApiTokenPayload(BaseModel):
|
||||
return v.lower().strip()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# NetBird client (peer) automatic updates
|
||||
# ---------------------------------------------------------------------------
|
||||
class NetbirdClientAutoUpdatePayload(BaseModel):
|
||||
"""Push a client auto-update version/mode to one or all customers."""
|
||||
|
||||
version: str = Field(..., max_length=50, description="'latest', 'disabled', or a version e.g. '0.61.0'")
|
||||
always: bool = False
|
||||
|
||||
|
||||
class NetbirdApiTokenPayload(BaseModel):
|
||||
"""Manually register a NetBird Personal Access Token for a customer.
|
||||
|
||||
Needed for customers deployed before automatic PAT capture existed —
|
||||
create a PAT once in that customer's own NetBird dashboard
|
||||
(Settings > Service Users / Personal Access Tokens) and paste it here.
|
||||
"""
|
||||
|
||||
token: str = Field(..., min_length=10, max_length=500)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Users
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -1260,6 +1260,19 @@
|
||||
<p class="text-muted mb-0" data-i18n="monitoring.clickCheckUpdates">Click "Check for Updates" to compare local images with Docker Hub.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Update Run Log -->
|
||||
<div class="card shadow-sm mt-4">
|
||||
<div class="card-header d-flex justify-content-between align-items-center">
|
||||
<span><i class="bi bi-clock-history me-2"></i><span data-i18n="monitoring.updateLogTitle">Update Log</span></span>
|
||||
<button class="btn btn-outline-secondary btn-sm" onclick="loadUpdateLog()" id="btn-refresh-update-log">
|
||||
<i class="bi bi-arrow-clockwise me-1"></i><span data-i18n="monitoring.refresh">Refresh</span>
|
||||
</button>
|
||||
</div>
|
||||
<div class="card-body" id="update-log-body">
|
||||
<p class="text-muted mb-0" data-i18n="common.loading">Loading...</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
+110
-2
@@ -605,7 +605,17 @@ async function loadCustomerNetbirdUpdates(id, hasToken) {
|
||||
try {
|
||||
const data = await api('GET', `/customers/${id}/netbird-updates`);
|
||||
const isCustom = data.version && data.version !== 'disabled' && data.version !== 'latest';
|
||||
|
||||
let tokenInfoHtml = '';
|
||||
if (data.token_renewed_at) {
|
||||
const renewedDate = new Date(data.token_renewed_at);
|
||||
const expiresDate = new Date(renewedDate);
|
||||
expiresDate.setDate(expiresDate.getDate() + 365);
|
||||
tokenInfoHtml = `<div class="small text-muted mb-2">${t('customer.nbuTokenRenewed', { date: renewedDate.toLocaleDateString() })} · ${t('customer.nbuTokenExpires', { date: expiresDate.toLocaleDateString() })}</div>`;
|
||||
}
|
||||
|
||||
container.innerHTML = `
|
||||
${tokenInfoHtml}
|
||||
<div class="row g-2 align-items-end">
|
||||
<div class="col-auto">
|
||||
<label class="form-label small mb-1">${t('customer.nbuVersion')}</label>
|
||||
@@ -625,12 +635,27 @@ async function loadCustomerNetbirdUpdates(id, hasToken) {
|
||||
<button class="btn btn-outline-secondary btn-sm" onclick="syncCustomerNetbirdFromMaster(${id})">${t('customer.nbuSyncMaster')}</button>
|
||||
</div>
|
||||
</div>
|
||||
<div id="nbu-result" class="small mt-2"></div>`;
|
||||
<div id="nbu-result" class="small mt-2"></div>
|
||||
<div class="mt-2">
|
||||
<a href="#" class="small" onclick="event.preventDefault(); showChangeNetbirdToken(${id})">${t('customer.nbuChangeToken')}</a>
|
||||
</div>
|
||||
<div id="nbu-change-token-area" class="mt-2 d-none"></div>`;
|
||||
} catch (err) {
|
||||
container.innerHTML = `<div class="alert alert-warning py-2 small mb-0">${esc(err.message)}</div>`;
|
||||
}
|
||||
}
|
||||
|
||||
function showChangeNetbirdToken(id) {
|
||||
const area = document.getElementById('nbu-change-token-area');
|
||||
area.classList.remove('d-none');
|
||||
area.innerHTML = `
|
||||
<div class="input-group input-group-sm">
|
||||
<input type="text" class="form-control" id="nbu-token-input" placeholder="${t('customer.nbuTokenPlaceholder')}">
|
||||
<button class="btn btn-outline-primary" onclick="saveCustomerNetbirdToken(${id})">${t('customer.nbuSaveToken')}</button>
|
||||
</div>
|
||||
<div id="nbu-token-result" class="small mt-1"></div>`;
|
||||
}
|
||||
|
||||
async function saveCustomerNetbirdToken(id) {
|
||||
const input = document.getElementById('nbu-token-input');
|
||||
const resultEl = document.getElementById('nbu-token-result');
|
||||
@@ -1918,7 +1943,90 @@ function togglePasswordVisibility(inputId) {
|
||||
// Monitoring
|
||||
// ---------------------------------------------------------------------------
|
||||
async function loadMonitoring() {
|
||||
await Promise.all([loadResources(), loadAllCustomerStatuses()]);
|
||||
await Promise.all([loadResources(), loadAllCustomerStatuses(), loadUpdateLog()]);
|
||||
}
|
||||
|
||||
const _updateLogStatusBadge = {
|
||||
success: 'bg-success',
|
||||
no_update: 'bg-secondary',
|
||||
partial: 'bg-warning text-dark',
|
||||
failed: 'bg-danger',
|
||||
running: 'bg-info',
|
||||
};
|
||||
|
||||
async function loadUpdateLog() {
|
||||
const body = document.getElementById('update-log-body');
|
||||
body.innerHTML = `<span class="spinner-border spinner-border-sm me-2"></span>${t('common.loading')}`;
|
||||
try {
|
||||
const logs = await api('GET', '/monitoring/update-logs?limit=30');
|
||||
if (logs.length === 0) {
|
||||
body.innerHTML = `<p class="text-muted mb-0">${t('monitoring.noUpdateLogs')}</p>`;
|
||||
return;
|
||||
}
|
||||
const rows = logs.map((l, i) => {
|
||||
const badgeClass = _updateLogStatusBadge[l.status] || 'bg-secondary';
|
||||
const typeLabel = l.run_type === 'scheduled' ? t('monitoring.logTypeScheduled') : t('monitoring.logTypeManual');
|
||||
const pullSummary = l.pull_attempted
|
||||
? (l.pull_success ? t('monitoring.logPullOk') : t('monitoring.logPullFailed'))
|
||||
: t('monitoring.logPullSkipped');
|
||||
const applySummary = l.apply_attempted
|
||||
? t('monitoring.logApplySummary', { updated: l.customers_updated ?? 0, total: l.customers_total ?? 0 })
|
||||
: t('monitoring.logApplySkipped');
|
||||
const detailId = `update-log-detail-${i}`;
|
||||
const hasDetails = (l.pull_details && Object.keys(l.pull_details).length > 0) ||
|
||||
(l.customer_details && l.customer_details.length > 0) || l.error;
|
||||
const detailsHtml = hasDetails ? `
|
||||
<div class="collapse mt-2" id="${detailId}">
|
||||
${l.error ? `<div class="alert alert-danger py-2 small mb-2">${esc(l.error)}</div>` : ''}
|
||||
${l.pull_details ? `
|
||||
<table class="table table-sm mb-2">
|
||||
<thead><tr><th>${t('monitoring.thImage')}</th><th>${t('monitoring.thStatus')}</th><th></th></tr></thead>
|
||||
<tbody>${Object.entries(l.pull_details).map(([img, r]) => `<tr>
|
||||
<td><code class="small">${esc(img)}</code></td>
|
||||
<td>${r.success ? `<span class="badge bg-success">OK</span>` : `<span class="badge bg-danger">Error</span>`}</td>
|
||||
<td class="small text-muted">${esc(r.error || '')}</td>
|
||||
</tr>`).join('')}</tbody>
|
||||
</table>` : ''}
|
||||
${l.customer_details && l.customer_details.length > 0 ? `
|
||||
<table class="table table-sm mb-0">
|
||||
<thead><tr><th>${t('monitoring.thName')}</th><th>${t('monitoring.thStatus')}</th><th></th></tr></thead>
|
||||
<tbody>${l.customer_details.map(r => `<tr>
|
||||
<td>${esc(r.customer_name)}</td>
|
||||
<td>${r.success ? `<span class="badge bg-success">OK</span>` : `<span class="badge bg-danger">Error</span>`}</td>
|
||||
<td class="small text-muted">${esc(r.error || '')}</td>
|
||||
</tr>`).join('')}</tbody>
|
||||
</table>` : ''}
|
||||
</div>` : '';
|
||||
return `<tr>
|
||||
<td class="small text-nowrap">${formatDate(l.started_at)}</td>
|
||||
<td><span class="badge bg-light text-dark border">${typeLabel}</span></td>
|
||||
<td><span class="badge ${badgeClass}">${t('monitoring.logStatus_' + l.status)}</span></td>
|
||||
<td class="small">${pullSummary}</td>
|
||||
<td class="small">${applySummary}</td>
|
||||
<td class="small text-muted">${esc(l.trigger || '')}</td>
|
||||
<td>${hasDetails ? `<button class="btn btn-sm btn-outline-secondary" data-bs-toggle="collapse" data-bs-target="#${detailId}"><i class="bi bi-chevron-down"></i></button>` : ''}</td>
|
||||
</tr>${detailsHtml ? `<tr class="d-none"></tr><tr><td colspan="7" class="p-0 border-0">${detailsHtml}</td></tr>` : ''}`;
|
||||
}).join('');
|
||||
body.innerHTML = `
|
||||
<div class="table-responsive">
|
||||
<table class="table table-sm mb-0">
|
||||
<thead class="table-light">
|
||||
<tr>
|
||||
<th>${t('monitoring.logTime')}</th>
|
||||
<th>${t('monitoring.logType')}</th>
|
||||
<th>${t('monitoring.thStatus')}</th>
|
||||
<th>${t('monitoring.logPull')}</th>
|
||||
<th>${t('monitoring.logApply')}</th>
|
||||
<th>${t('monitoring.logTrigger')}</th>
|
||||
<th></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>${rows}</tbody>
|
||||
</table>
|
||||
</div>`;
|
||||
} catch (err) {
|
||||
body.innerHTML = `<div class="alert alert-danger">${esc(err.message)}</div>`;
|
||||
}
|
||||
}
|
||||
|
||||
async function loadResources() {
|
||||
|
||||
+23
-1
@@ -97,6 +97,9 @@
|
||||
"nbuTokenPlaceholder": "Personal Access Token einfügen…",
|
||||
"nbuSaveToken": "Prüfen & Speichern",
|
||||
"nbuTokenSaved": "Token gespeichert.",
|
||||
"nbuChangeToken": "Token ändern",
|
||||
"nbuTokenRenewed": "Token erneuert am {date}",
|
||||
"nbuTokenExpires": "gültig bis {date}",
|
||||
"nbuVersion": "Client-Version",
|
||||
"nbuDisabled": "Deaktiviert",
|
||||
"nbuLatest": "Neueste Version",
|
||||
@@ -451,6 +454,25 @@
|
||||
"autoUpdateApplyHint": "Wenn aktiviert, werden nach einer gefundenen Aktualisierung automatisch alle Kunden-Container neu erstellt (kurzer Neustart der Dienste). Wenn deaktiviert, werden nur die Images geladen — die Aktualisierung der Kunden erfolgt weiterhin manuell.",
|
||||
"autoUpdateLastRun": "Letzte automatische Prüfung",
|
||||
"autoUpdateNever": "Noch nie ausgeführt",
|
||||
"saveAutoUpdateSettings": "Automatisierung speichern"
|
||||
"saveAutoUpdateSettings": "Automatisierung speichern",
|
||||
"updateLogTitle": "Update-Verlauf",
|
||||
"noUpdateLogs": "Noch keine Update-Läufe protokolliert.",
|
||||
"logTypeScheduled": "Automatisch",
|
||||
"logTypeManual": "Manuell",
|
||||
"logTime": "Zeitpunkt",
|
||||
"logType": "Typ",
|
||||
"logPull": "Image-Download",
|
||||
"logApply": "Kunden-Update",
|
||||
"logTrigger": "Ausgelöst von",
|
||||
"logPullOk": "Erfolgreich",
|
||||
"logPullFailed": "Fehlgeschlagen",
|
||||
"logPullSkipped": "Nicht nötig",
|
||||
"logApplySummary": "{updated}/{total} Kunden",
|
||||
"logApplySkipped": "Nicht ausgeführt",
|
||||
"logStatus_success": "Erfolgreich",
|
||||
"logStatus_no_update": "Kein Update nötig",
|
||||
"logStatus_partial": "Teilweise",
|
||||
"logStatus_failed": "Fehlgeschlagen",
|
||||
"logStatus_running": "Läuft…"
|
||||
}
|
||||
}
|
||||
+23
-1
@@ -97,6 +97,9 @@
|
||||
"nbuTokenPlaceholder": "Paste Personal Access Token…",
|
||||
"nbuSaveToken": "Verify & Save",
|
||||
"nbuTokenSaved": "Token saved.",
|
||||
"nbuChangeToken": "Change token",
|
||||
"nbuTokenRenewed": "Token renewed on {date}",
|
||||
"nbuTokenExpires": "valid until {date}",
|
||||
"nbuVersion": "Client version",
|
||||
"nbuDisabled": "Disabled",
|
||||
"nbuLatest": "Latest version",
|
||||
@@ -358,7 +361,26 @@
|
||||
"autoUpdateApplyHint": "When enabled, all customer containers are automatically recreated after a new image is found (services briefly restart). When disabled, only the images are pulled — updating customers stays a manual step.",
|
||||
"autoUpdateLastRun": "Last automatic check",
|
||||
"autoUpdateNever": "Never run",
|
||||
"saveAutoUpdateSettings": "Save Automation"
|
||||
"saveAutoUpdateSettings": "Save Automation",
|
||||
"updateLogTitle": "Update Log",
|
||||
"noUpdateLogs": "No update runs logged yet.",
|
||||
"logTypeScheduled": "Automatic",
|
||||
"logTypeManual": "Manual",
|
||||
"logTime": "Time",
|
||||
"logType": "Type",
|
||||
"logPull": "Image Pull",
|
||||
"logApply": "Customer Update",
|
||||
"logTrigger": "Triggered by",
|
||||
"logPullOk": "Succeeded",
|
||||
"logPullFailed": "Failed",
|
||||
"logPullSkipped": "Not needed",
|
||||
"logApplySummary": "{updated}/{total} customers",
|
||||
"logApplySkipped": "Not run",
|
||||
"logStatus_success": "Success",
|
||||
"logStatus_no_update": "No update needed",
|
||||
"logStatus_partial": "Partial",
|
||||
"logStatus_failed": "Failed",
|
||||
"logStatus_running": "Running…"
|
||||
},
|
||||
"userModal": {
|
||||
"title": "New User",
|
||||
|
||||
Reference in New Issue
Block a user