feat(netbird): auto-renew NetBird client-update API tokens before expiry

The tokens captured for central update control expire (NetBird enforces a
365-day max on Personal Access Tokens), and nothing was renewing them —
discovered that the 47 tokens created via the browser-automation bulk
onboarding were actually only 30-day tokens (left the UI's default
expiration field untouched instead of setting 365), so they would have
silently broken automatic-update control next month with no warning.

- Bumped all existing tokens to fresh 365-day ones via the API (using the
  still-valid old token as bearer — no re-login needed)
- Added netbird_api_token_renewed_at per deployment
- Scheduler now checks daily and renews any token older than 300 days
  automatically, so this never needs to be done by hand again
This commit is contained in:
2026-08-19 11:15:49 +02:00
parent 8a84e60a27
commit e9afdb226c
6 changed files with 110 additions and 5 deletions
+1
View File
@@ -129,6 +129,7 @@ def _run_migrations() -> None:
("system_config", "auto_update_last_run_at", "TEXT"), ("system_config", "auto_update_last_run_at", "TEXT"),
# NetBird client (peer) automatic-updates master default + per-customer PAT # NetBird client (peer) automatic-updates master default + per-customer PAT
("deployments", "netbird_api_token_encrypted", "TEXT"), ("deployments", "netbird_api_token_encrypted", "TEXT"),
("deployments", "netbird_api_token_renewed_at", "TEXT"),
("system_config", "netbird_client_auto_update_version", "TEXT DEFAULT 'disabled'"), ("system_config", "netbird_client_auto_update_version", "TEXT DEFAULT 'disabled'"),
("system_config", "netbird_client_auto_update_always", "BOOLEAN DEFAULT 0"), ("system_config", "netbird_client_auto_update_always", "BOOLEAN DEFAULT 0"),
] ]
+4
View File
@@ -89,6 +89,7 @@ class Deployment(Base):
netbird_admin_email: Mapped[Optional[str]] = mapped_column(Text, nullable=True) netbird_admin_email: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
netbird_admin_password: Mapped[Optional[str]] = mapped_column(Text, nullable=True) netbird_admin_password: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
netbird_api_token_encrypted: Mapped[Optional[str]] = mapped_column(Text, nullable=True) netbird_api_token_encrypted: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
netbird_api_token_renewed_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
deployment_status: Mapped[str] = mapped_column( deployment_status: Mapped[str] = mapped_column(
String(20), default="pending", nullable=False String(20), default="pending", nullable=False
) )
@@ -118,6 +119,9 @@ class Deployment(Base):
"setup_url": self.setup_url, "setup_url": self.setup_url,
"has_credentials": bool(self.netbird_admin_email and self.netbird_admin_password), "has_credentials": bool(self.netbird_admin_email and self.netbird_admin_password),
"has_netbird_api_token": bool(self.netbird_api_token_encrypted), "has_netbird_api_token": bool(self.netbird_api_token_encrypted),
"netbird_api_token_renewed_at": (
self.netbird_api_token_renewed_at.isoformat() if self.netbird_api_token_renewed_at else None
),
"deployment_status": self.deployment_status, "deployment_status": self.deployment_status,
"deployed_at": self.deployed_at.isoformat() if self.deployed_at else None, "deployed_at": self.deployed_at.isoformat() if self.deployed_at else None,
"last_health_check": ( "last_health_check": (
+2
View File
@@ -1,6 +1,7 @@
"""Deployment management API — start, stop, restart, logs, health for customers.""" """Deployment management API — start, stop, restart, logs, health for customers."""
import logging import logging
from datetime import datetime
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query, status from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query, status
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
@@ -362,6 +363,7 @@ async def set_customer_netbird_api_token(
) )
deployment.netbird_api_token_encrypted = encrypt_value(payload.token) deployment.netbird_api_token_encrypted = encrypt_value(payload.token)
deployment.netbird_api_token_renewed_at = datetime.utcnow()
db.commit() db.commit()
logger.info("NetBird API token registered for customer %d by %s.", customer_id, current_user.username) logger.info("NetBird API token registered for customer %d by %s.", customer_id, current_user.username)
return {"ok": True} return {"ok": True}
@@ -52,6 +52,48 @@ async def get_current_settings(container_prefix: str, token: str) -> dict[str, A
return {"ok": False, "error": str(exc)} return {"ok": False, "error": str(exc)}
async def renew_token(container_prefix: str, token: str) -> dict[str, Any]:
"""Mint a fresh 365-day Personal Access Token using the current one.
The old token is left in place (it naturally expires and NetBird gives no
reliable way to identify "our" token among a user's other PATs by content
alone, only by name — which isn't safe to assume is unique). One
harmless unused token lingering until its own expiry is an acceptable
trade-off for not risking deleting a token that turns out to be in use.
"""
base_url = _base_url(container_prefix)
headers = {"Authorization": f"Token {token}"}
try:
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
resp = await client.get(f"{base_url}/api/users", headers=headers)
if resp.status_code != 200:
return {"ok": False, "error": f"GET /api/users -> HTTP {resp.status_code}"}
users = resp.json()
me = next((u for u in users if u.get("is_current")), None)
if not me:
return {"ok": False, "error": "Could not identify current user from /api/users."}
create_resp = await client.post(
f"{base_url}/api/users/{me['id']}/tokens",
headers={**headers, "Content-Type": "application/json"},
content=json.dumps({"name": "MSP Central Management", "expires_in": 365}),
)
if create_resp.status_code not in (200, 201):
return {"ok": False, "error": f"POST tokens -> HTTP {create_resp.status_code}: {create_resp.text[:200]}"}
new_token = create_resp.json().get("plain_token")
if not new_token:
return {"ok": False, "error": "Token creation response had no plain_token."}
verify_resp = await client.get(f"{base_url}/api/accounts", headers={"Authorization": f"Token {new_token}"})
if verify_resp.status_code != 200:
return {"ok": False, "error": "New token failed verification."}
return {"ok": True, "token": new_token}
except Exception as exc:
logger.warning("Failed to renew NetBird API token for %s: %s", container_prefix, exc)
return {"ok": False, "error": str(exc)}
async def push_auto_update_settings( async def push_auto_update_settings(
container_prefix: str, token: str, version: str, always: bool container_prefix: str, token: str, version: str, always: bool
) -> dict[str, Any]: ) -> dict[str, Any]:
+4 -3
View File
@@ -348,9 +348,9 @@ async def deploy_customer(db: Session, customer_id: int) -> dict[str, Any]:
deployment.setup_url = setup_url deployment.setup_url = setup_url
deployment.netbird_admin_email = encrypt_value(admin_email) if setup_ok else deployment.netbird_admin_email deployment.netbird_admin_email = encrypt_value(admin_email) if setup_ok else deployment.netbird_admin_email
deployment.netbird_admin_password = encrypt_value(admin_password) if setup_ok else deployment.netbird_admin_password deployment.netbird_admin_password = encrypt_value(admin_password) if setup_ok else deployment.netbird_admin_password
deployment.netbird_api_token_encrypted = ( if netbird_api_token:
encrypt_value(netbird_api_token) if netbird_api_token else deployment.netbird_api_token_encrypted deployment.netbird_api_token_encrypted = encrypt_value(netbird_api_token)
) deployment.netbird_api_token_renewed_at = datetime.utcnow()
deployment.deployment_status = "running" deployment.deployment_status = "running"
deployment.deployed_at = datetime.utcnow() deployment.deployed_at = datetime.utcnow()
else: else:
@@ -366,6 +366,7 @@ async def deploy_customer(db: Session, customer_id: int) -> dict[str, Any]:
netbird_admin_email=encrypt_value(admin_email) if setup_ok else None, netbird_admin_email=encrypt_value(admin_email) if setup_ok else None,
netbird_admin_password=encrypt_value(admin_password) if setup_ok else None, netbird_admin_password=encrypt_value(admin_password) if setup_ok else None,
netbird_api_token_encrypted=encrypt_value(netbird_api_token) if netbird_api_token else None, netbird_api_token_encrypted=encrypt_value(netbird_api_token) if netbird_api_token else None,
netbird_api_token_renewed_at=datetime.utcnow() if netbird_api_token else None,
deployment_status="running", deployment_status="running",
deployed_at=datetime.utcnow(), deployed_at=datetime.utcnow(),
) )
+57 -2
View File
@@ -8,17 +8,23 @@ SystemConfig.auto_update_check_enabled / auto_update_check_time.
import asyncio import asyncio
import logging import logging
from datetime import datetime from datetime import datetime, timedelta
from app.database import SessionLocal from app.database import SessionLocal
from app.models import Deployment, SystemConfig from app.models import Deployment, SystemConfig
from app.services import image_service from app.services import image_service, netbird_client_update_service
from app.utils.security import decrypt_value, encrypt_value
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
_POLL_INTERVAL_SECONDS = 60 _POLL_INTERVAL_SECONDS = 60
_task: asyncio.Task | None = None _task: asyncio.Task | None = None
# NetBird PATs we mint are issued for 365 days; renew well before that so a
# missed tick or a slow rollout never risks the token actually expiring.
_TOKEN_RENEW_AFTER_DAYS = 300
_TOKEN_RENEW_CHECK_HOUR = 4 # run once per day, distinct from the image-check hour
def start() -> None: def start() -> None:
"""Start the background polling task. Safe to call once at app startup.""" """Start the background polling task. Safe to call once at app startup."""
@@ -36,15 +42,64 @@ def stop() -> None:
_task = None _task = None
_last_token_renewal_date = None
async def _poll_loop() -> None: async def _poll_loop() -> None:
while True: while True:
try: try:
await _tick() await _tick()
except Exception: except Exception:
logger.exception("Scheduler tick failed") logger.exception("Scheduler tick failed")
try:
await _token_renewal_tick()
except Exception:
logger.exception("Token renewal tick failed")
await asyncio.sleep(_POLL_INTERVAL_SECONDS) await asyncio.sleep(_POLL_INTERVAL_SECONDS)
async def _token_renewal_tick() -> None:
"""Once a day, renew any NetBird client-update API token nearing its
365-day expiry — keeps central update control working indefinitely
without anyone needing to notice or act.
"""
global _last_token_renewal_date
now = datetime.now()
if now.hour != _TOKEN_RENEW_CHECK_HOUR:
return
if _last_token_renewal_date == now.date():
return
_last_token_renewal_date = now.date()
db = SessionLocal()
try:
cutoff = now - timedelta(days=_TOKEN_RENEW_AFTER_DAYS)
deployments = (
db.query(Deployment)
.filter(Deployment.netbird_api_token_encrypted.isnot(None))
.all()
)
due = [
d for d in deployments
if d.netbird_api_token_renewed_at is None or d.netbird_api_token_renewed_at < cutoff
]
if not due:
return
logger.info("Renewing NetBird API token for %d customer(s)...", len(due))
for d in due:
token = decrypt_value(d.netbird_api_token_encrypted)
result = await netbird_client_update_service.renew_token(d.container_prefix, token)
if result["ok"]:
d.netbird_api_token_encrypted = encrypt_value(result["token"])
d.netbird_api_token_renewed_at = now
db.commit()
logger.info("Renewed NetBird API token for %s.", d.container_prefix)
else:
logger.warning("Failed to renew NetBird API token for %s: %s", d.container_prefix, result.get("error"))
finally:
db.close()
async def _tick() -> None: async def _tick() -> None:
db = SessionLocal() db = SessionLocal()
try: try: