feat(netbird): auto-renew NetBird client-update API tokens before expiry
The tokens captured for central update control expire (NetBird enforces a 365-day max on Personal Access Tokens), and nothing was renewing them — discovered that the 47 tokens created via the browser-automation bulk onboarding were actually only 30-day tokens (left the UI's default expiration field untouched instead of setting 365), so they would have silently broken automatic-update control next month with no warning. - Bumped all existing tokens to fresh 365-day ones via the API (using the still-valid old token as bearer — no re-login needed) - Added netbird_api_token_renewed_at per deployment - Scheduler now checks daily and renews any token older than 300 days automatically, so this never needs to be done by hand again
This commit is contained in:
@@ -129,6 +129,7 @@ def _run_migrations() -> None:
|
||||
("system_config", "auto_update_last_run_at", "TEXT"),
|
||||
# NetBird client (peer) automatic-updates master default + per-customer PAT
|
||||
("deployments", "netbird_api_token_encrypted", "TEXT"),
|
||||
("deployments", "netbird_api_token_renewed_at", "TEXT"),
|
||||
("system_config", "netbird_client_auto_update_version", "TEXT DEFAULT 'disabled'"),
|
||||
("system_config", "netbird_client_auto_update_always", "BOOLEAN DEFAULT 0"),
|
||||
]
|
||||
|
||||
@@ -89,6 +89,7 @@ class Deployment(Base):
|
||||
netbird_admin_email: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||
netbird_admin_password: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||
netbird_api_token_encrypted: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||
netbird_api_token_renewed_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
|
||||
deployment_status: Mapped[str] = mapped_column(
|
||||
String(20), default="pending", nullable=False
|
||||
)
|
||||
@@ -118,6 +119,9 @@ class Deployment(Base):
|
||||
"setup_url": self.setup_url,
|
||||
"has_credentials": bool(self.netbird_admin_email and self.netbird_admin_password),
|
||||
"has_netbird_api_token": bool(self.netbird_api_token_encrypted),
|
||||
"netbird_api_token_renewed_at": (
|
||||
self.netbird_api_token_renewed_at.isoformat() if self.netbird_api_token_renewed_at else None
|
||||
),
|
||||
"deployment_status": self.deployment_status,
|
||||
"deployed_at": self.deployed_at.isoformat() if self.deployed_at else None,
|
||||
"last_health_check": (
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Deployment management API — start, stop, restart, logs, health for customers."""
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query, status
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -362,6 +363,7 @@ async def set_customer_netbird_api_token(
|
||||
)
|
||||
|
||||
deployment.netbird_api_token_encrypted = encrypt_value(payload.token)
|
||||
deployment.netbird_api_token_renewed_at = datetime.utcnow()
|
||||
db.commit()
|
||||
logger.info("NetBird API token registered for customer %d by %s.", customer_id, current_user.username)
|
||||
return {"ok": True}
|
||||
|
||||
@@ -52,6 +52,48 @@ async def get_current_settings(container_prefix: str, token: str) -> dict[str, A
|
||||
return {"ok": False, "error": str(exc)}
|
||||
|
||||
|
||||
async def renew_token(container_prefix: str, token: str) -> dict[str, Any]:
|
||||
"""Mint a fresh 365-day Personal Access Token using the current one.
|
||||
|
||||
The old token is left in place (it naturally expires and NetBird gives no
|
||||
reliable way to identify "our" token among a user's other PATs by content
|
||||
alone, only by name — which isn't safe to assume is unique). One
|
||||
harmless unused token lingering until its own expiry is an acceptable
|
||||
trade-off for not risking deleting a token that turns out to be in use.
|
||||
"""
|
||||
base_url = _base_url(container_prefix)
|
||||
headers = {"Authorization": f"Token {token}"}
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
|
||||
resp = await client.get(f"{base_url}/api/users", headers=headers)
|
||||
if resp.status_code != 200:
|
||||
return {"ok": False, "error": f"GET /api/users -> HTTP {resp.status_code}"}
|
||||
users = resp.json()
|
||||
me = next((u for u in users if u.get("is_current")), None)
|
||||
if not me:
|
||||
return {"ok": False, "error": "Could not identify current user from /api/users."}
|
||||
|
||||
create_resp = await client.post(
|
||||
f"{base_url}/api/users/{me['id']}/tokens",
|
||||
headers={**headers, "Content-Type": "application/json"},
|
||||
content=json.dumps({"name": "MSP Central Management", "expires_in": 365}),
|
||||
)
|
||||
if create_resp.status_code not in (200, 201):
|
||||
return {"ok": False, "error": f"POST tokens -> HTTP {create_resp.status_code}: {create_resp.text[:200]}"}
|
||||
new_token = create_resp.json().get("plain_token")
|
||||
if not new_token:
|
||||
return {"ok": False, "error": "Token creation response had no plain_token."}
|
||||
|
||||
verify_resp = await client.get(f"{base_url}/api/accounts", headers={"Authorization": f"Token {new_token}"})
|
||||
if verify_resp.status_code != 200:
|
||||
return {"ok": False, "error": "New token failed verification."}
|
||||
|
||||
return {"ok": True, "token": new_token}
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to renew NetBird API token for %s: %s", container_prefix, exc)
|
||||
return {"ok": False, "error": str(exc)}
|
||||
|
||||
|
||||
async def push_auto_update_settings(
|
||||
container_prefix: str, token: str, version: str, always: bool
|
||||
) -> dict[str, Any]:
|
||||
|
||||
@@ -348,9 +348,9 @@ async def deploy_customer(db: Session, customer_id: int) -> dict[str, Any]:
|
||||
deployment.setup_url = setup_url
|
||||
deployment.netbird_admin_email = encrypt_value(admin_email) if setup_ok else deployment.netbird_admin_email
|
||||
deployment.netbird_admin_password = encrypt_value(admin_password) if setup_ok else deployment.netbird_admin_password
|
||||
deployment.netbird_api_token_encrypted = (
|
||||
encrypt_value(netbird_api_token) if netbird_api_token else deployment.netbird_api_token_encrypted
|
||||
)
|
||||
if netbird_api_token:
|
||||
deployment.netbird_api_token_encrypted = encrypt_value(netbird_api_token)
|
||||
deployment.netbird_api_token_renewed_at = datetime.utcnow()
|
||||
deployment.deployment_status = "running"
|
||||
deployment.deployed_at = datetime.utcnow()
|
||||
else:
|
||||
@@ -366,6 +366,7 @@ async def deploy_customer(db: Session, customer_id: int) -> dict[str, Any]:
|
||||
netbird_admin_email=encrypt_value(admin_email) if setup_ok else None,
|
||||
netbird_admin_password=encrypt_value(admin_password) if setup_ok else None,
|
||||
netbird_api_token_encrypted=encrypt_value(netbird_api_token) if netbird_api_token else None,
|
||||
netbird_api_token_renewed_at=datetime.utcnow() if netbird_api_token else None,
|
||||
deployment_status="running",
|
||||
deployed_at=datetime.utcnow(),
|
||||
)
|
||||
|
||||
@@ -8,17 +8,23 @@ SystemConfig.auto_update_check_enabled / auto_update_check_time.
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from app.database import SessionLocal
|
||||
from app.models import Deployment, SystemConfig
|
||||
from app.services import image_service
|
||||
from app.services import image_service, netbird_client_update_service
|
||||
from app.utils.security import decrypt_value, encrypt_value
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_POLL_INTERVAL_SECONDS = 60
|
||||
_task: asyncio.Task | None = None
|
||||
|
||||
# NetBird PATs we mint are issued for 365 days; renew well before that so a
|
||||
# missed tick or a slow rollout never risks the token actually expiring.
|
||||
_TOKEN_RENEW_AFTER_DAYS = 300
|
||||
_TOKEN_RENEW_CHECK_HOUR = 4 # run once per day, distinct from the image-check hour
|
||||
|
||||
|
||||
def start() -> None:
|
||||
"""Start the background polling task. Safe to call once at app startup."""
|
||||
@@ -36,15 +42,64 @@ def stop() -> None:
|
||||
_task = None
|
||||
|
||||
|
||||
_last_token_renewal_date = None
|
||||
|
||||
|
||||
async def _poll_loop() -> None:
|
||||
while True:
|
||||
try:
|
||||
await _tick()
|
||||
except Exception:
|
||||
logger.exception("Scheduler tick failed")
|
||||
try:
|
||||
await _token_renewal_tick()
|
||||
except Exception:
|
||||
logger.exception("Token renewal tick failed")
|
||||
await asyncio.sleep(_POLL_INTERVAL_SECONDS)
|
||||
|
||||
|
||||
async def _token_renewal_tick() -> None:
|
||||
"""Once a day, renew any NetBird client-update API token nearing its
|
||||
365-day expiry — keeps central update control working indefinitely
|
||||
without anyone needing to notice or act.
|
||||
"""
|
||||
global _last_token_renewal_date
|
||||
now = datetime.now()
|
||||
if now.hour != _TOKEN_RENEW_CHECK_HOUR:
|
||||
return
|
||||
if _last_token_renewal_date == now.date():
|
||||
return
|
||||
_last_token_renewal_date = now.date()
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
cutoff = now - timedelta(days=_TOKEN_RENEW_AFTER_DAYS)
|
||||
deployments = (
|
||||
db.query(Deployment)
|
||||
.filter(Deployment.netbird_api_token_encrypted.isnot(None))
|
||||
.all()
|
||||
)
|
||||
due = [
|
||||
d for d in deployments
|
||||
if d.netbird_api_token_renewed_at is None or d.netbird_api_token_renewed_at < cutoff
|
||||
]
|
||||
if not due:
|
||||
return
|
||||
logger.info("Renewing NetBird API token for %d customer(s)...", len(due))
|
||||
for d in due:
|
||||
token = decrypt_value(d.netbird_api_token_encrypted)
|
||||
result = await netbird_client_update_service.renew_token(d.container_prefix, token)
|
||||
if result["ok"]:
|
||||
d.netbird_api_token_encrypted = encrypt_value(result["token"])
|
||||
d.netbird_api_token_renewed_at = now
|
||||
db.commit()
|
||||
logger.info("Renewed NetBird API token for %s.", d.container_prefix)
|
||||
else:
|
||||
logger.warning("Failed to renew NetBird API token for %s: %s", d.container_prefix, result.get("error"))
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
async def _tick() -> None:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user