feat(netbird): auto-renew NetBird client-update API tokens before expiry
The tokens captured for central update control expire (NetBird enforces a 365-day max on Personal Access Tokens), and nothing was renewing them — discovered that the 47 tokens created via the browser-automation bulk onboarding were actually only 30-day tokens (left the UI's default expiration field untouched instead of setting 365), so they would have silently broken automatic-update control next month with no warning. - Bumped all existing tokens to fresh 365-day ones via the API (using the still-valid old token as bearer — no re-login needed) - Added netbird_api_token_renewed_at per deployment - Scheduler now checks daily and renews any token older than 300 days automatically, so this never needs to be done by hand again
This commit is contained in:
@@ -129,6 +129,7 @@ def _run_migrations() -> None:
|
|||||||
("system_config", "auto_update_last_run_at", "TEXT"),
|
("system_config", "auto_update_last_run_at", "TEXT"),
|
||||||
# NetBird client (peer) automatic-updates master default + per-customer PAT
|
# NetBird client (peer) automatic-updates master default + per-customer PAT
|
||||||
("deployments", "netbird_api_token_encrypted", "TEXT"),
|
("deployments", "netbird_api_token_encrypted", "TEXT"),
|
||||||
|
("deployments", "netbird_api_token_renewed_at", "TEXT"),
|
||||||
("system_config", "netbird_client_auto_update_version", "TEXT DEFAULT 'disabled'"),
|
("system_config", "netbird_client_auto_update_version", "TEXT DEFAULT 'disabled'"),
|
||||||
("system_config", "netbird_client_auto_update_always", "BOOLEAN DEFAULT 0"),
|
("system_config", "netbird_client_auto_update_always", "BOOLEAN DEFAULT 0"),
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -89,6 +89,7 @@ class Deployment(Base):
|
|||||||
netbird_admin_email: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
netbird_admin_email: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||||
netbird_admin_password: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
netbird_admin_password: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||||
netbird_api_token_encrypted: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
netbird_api_token_encrypted: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||||
|
netbird_api_token_renewed_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
|
||||||
deployment_status: Mapped[str] = mapped_column(
|
deployment_status: Mapped[str] = mapped_column(
|
||||||
String(20), default="pending", nullable=False
|
String(20), default="pending", nullable=False
|
||||||
)
|
)
|
||||||
@@ -118,6 +119,9 @@ class Deployment(Base):
|
|||||||
"setup_url": self.setup_url,
|
"setup_url": self.setup_url,
|
||||||
"has_credentials": bool(self.netbird_admin_email and self.netbird_admin_password),
|
"has_credentials": bool(self.netbird_admin_email and self.netbird_admin_password),
|
||||||
"has_netbird_api_token": bool(self.netbird_api_token_encrypted),
|
"has_netbird_api_token": bool(self.netbird_api_token_encrypted),
|
||||||
|
"netbird_api_token_renewed_at": (
|
||||||
|
self.netbird_api_token_renewed_at.isoformat() if self.netbird_api_token_renewed_at else None
|
||||||
|
),
|
||||||
"deployment_status": self.deployment_status,
|
"deployment_status": self.deployment_status,
|
||||||
"deployed_at": self.deployed_at.isoformat() if self.deployed_at else None,
|
"deployed_at": self.deployed_at.isoformat() if self.deployed_at else None,
|
||||||
"last_health_check": (
|
"last_health_check": (
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
"""Deployment management API — start, stop, restart, logs, health for customers."""
|
"""Deployment management API — start, stop, restart, logs, health for customers."""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query, status
|
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query, status
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
@@ -362,6 +363,7 @@ async def set_customer_netbird_api_token(
|
|||||||
)
|
)
|
||||||
|
|
||||||
deployment.netbird_api_token_encrypted = encrypt_value(payload.token)
|
deployment.netbird_api_token_encrypted = encrypt_value(payload.token)
|
||||||
|
deployment.netbird_api_token_renewed_at = datetime.utcnow()
|
||||||
db.commit()
|
db.commit()
|
||||||
logger.info("NetBird API token registered for customer %d by %s.", customer_id, current_user.username)
|
logger.info("NetBird API token registered for customer %d by %s.", customer_id, current_user.username)
|
||||||
return {"ok": True}
|
return {"ok": True}
|
||||||
|
|||||||
@@ -52,6 +52,48 @@ async def get_current_settings(container_prefix: str, token: str) -> dict[str, A
|
|||||||
return {"ok": False, "error": str(exc)}
|
return {"ok": False, "error": str(exc)}
|
||||||
|
|
||||||
|
|
||||||
|
async def renew_token(container_prefix: str, token: str) -> dict[str, Any]:
|
||||||
|
"""Mint a fresh 365-day Personal Access Token using the current one.
|
||||||
|
|
||||||
|
The old token is left in place (it naturally expires and NetBird gives no
|
||||||
|
reliable way to identify "our" token among a user's other PATs by content
|
||||||
|
alone, only by name — which isn't safe to assume is unique). One
|
||||||
|
harmless unused token lingering until its own expiry is an acceptable
|
||||||
|
trade-off for not risking deleting a token that turns out to be in use.
|
||||||
|
"""
|
||||||
|
base_url = _base_url(container_prefix)
|
||||||
|
headers = {"Authorization": f"Token {token}"}
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
|
||||||
|
resp = await client.get(f"{base_url}/api/users", headers=headers)
|
||||||
|
if resp.status_code != 200:
|
||||||
|
return {"ok": False, "error": f"GET /api/users -> HTTP {resp.status_code}"}
|
||||||
|
users = resp.json()
|
||||||
|
me = next((u for u in users if u.get("is_current")), None)
|
||||||
|
if not me:
|
||||||
|
return {"ok": False, "error": "Could not identify current user from /api/users."}
|
||||||
|
|
||||||
|
create_resp = await client.post(
|
||||||
|
f"{base_url}/api/users/{me['id']}/tokens",
|
||||||
|
headers={**headers, "Content-Type": "application/json"},
|
||||||
|
content=json.dumps({"name": "MSP Central Management", "expires_in": 365}),
|
||||||
|
)
|
||||||
|
if create_resp.status_code not in (200, 201):
|
||||||
|
return {"ok": False, "error": f"POST tokens -> HTTP {create_resp.status_code}: {create_resp.text[:200]}"}
|
||||||
|
new_token = create_resp.json().get("plain_token")
|
||||||
|
if not new_token:
|
||||||
|
return {"ok": False, "error": "Token creation response had no plain_token."}
|
||||||
|
|
||||||
|
verify_resp = await client.get(f"{base_url}/api/accounts", headers={"Authorization": f"Token {new_token}"})
|
||||||
|
if verify_resp.status_code != 200:
|
||||||
|
return {"ok": False, "error": "New token failed verification."}
|
||||||
|
|
||||||
|
return {"ok": True, "token": new_token}
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("Failed to renew NetBird API token for %s: %s", container_prefix, exc)
|
||||||
|
return {"ok": False, "error": str(exc)}
|
||||||
|
|
||||||
|
|
||||||
async def push_auto_update_settings(
|
async def push_auto_update_settings(
|
||||||
container_prefix: str, token: str, version: str, always: bool
|
container_prefix: str, token: str, version: str, always: bool
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
|
|||||||
@@ -348,9 +348,9 @@ async def deploy_customer(db: Session, customer_id: int) -> dict[str, Any]:
|
|||||||
deployment.setup_url = setup_url
|
deployment.setup_url = setup_url
|
||||||
deployment.netbird_admin_email = encrypt_value(admin_email) if setup_ok else deployment.netbird_admin_email
|
deployment.netbird_admin_email = encrypt_value(admin_email) if setup_ok else deployment.netbird_admin_email
|
||||||
deployment.netbird_admin_password = encrypt_value(admin_password) if setup_ok else deployment.netbird_admin_password
|
deployment.netbird_admin_password = encrypt_value(admin_password) if setup_ok else deployment.netbird_admin_password
|
||||||
deployment.netbird_api_token_encrypted = (
|
if netbird_api_token:
|
||||||
encrypt_value(netbird_api_token) if netbird_api_token else deployment.netbird_api_token_encrypted
|
deployment.netbird_api_token_encrypted = encrypt_value(netbird_api_token)
|
||||||
)
|
deployment.netbird_api_token_renewed_at = datetime.utcnow()
|
||||||
deployment.deployment_status = "running"
|
deployment.deployment_status = "running"
|
||||||
deployment.deployed_at = datetime.utcnow()
|
deployment.deployed_at = datetime.utcnow()
|
||||||
else:
|
else:
|
||||||
@@ -366,6 +366,7 @@ async def deploy_customer(db: Session, customer_id: int) -> dict[str, Any]:
|
|||||||
netbird_admin_email=encrypt_value(admin_email) if setup_ok else None,
|
netbird_admin_email=encrypt_value(admin_email) if setup_ok else None,
|
||||||
netbird_admin_password=encrypt_value(admin_password) if setup_ok else None,
|
netbird_admin_password=encrypt_value(admin_password) if setup_ok else None,
|
||||||
netbird_api_token_encrypted=encrypt_value(netbird_api_token) if netbird_api_token else None,
|
netbird_api_token_encrypted=encrypt_value(netbird_api_token) if netbird_api_token else None,
|
||||||
|
netbird_api_token_renewed_at=datetime.utcnow() if netbird_api_token else None,
|
||||||
deployment_status="running",
|
deployment_status="running",
|
||||||
deployed_at=datetime.utcnow(),
|
deployed_at=datetime.utcnow(),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -8,17 +8,23 @@ SystemConfig.auto_update_check_enabled / auto_update_check_time.
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
from datetime import datetime
|
from datetime import datetime, timedelta
|
||||||
|
|
||||||
from app.database import SessionLocal
|
from app.database import SessionLocal
|
||||||
from app.models import Deployment, SystemConfig
|
from app.models import Deployment, SystemConfig
|
||||||
from app.services import image_service
|
from app.services import image_service, netbird_client_update_service
|
||||||
|
from app.utils.security import decrypt_value, encrypt_value
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
_POLL_INTERVAL_SECONDS = 60
|
_POLL_INTERVAL_SECONDS = 60
|
||||||
_task: asyncio.Task | None = None
|
_task: asyncio.Task | None = None
|
||||||
|
|
||||||
|
# NetBird PATs we mint are issued for 365 days; renew well before that so a
|
||||||
|
# missed tick or a slow rollout never risks the token actually expiring.
|
||||||
|
_TOKEN_RENEW_AFTER_DAYS = 300
|
||||||
|
_TOKEN_RENEW_CHECK_HOUR = 4 # run once per day, distinct from the image-check hour
|
||||||
|
|
||||||
|
|
||||||
def start() -> None:
|
def start() -> None:
|
||||||
"""Start the background polling task. Safe to call once at app startup."""
|
"""Start the background polling task. Safe to call once at app startup."""
|
||||||
@@ -36,15 +42,64 @@ def stop() -> None:
|
|||||||
_task = None
|
_task = None
|
||||||
|
|
||||||
|
|
||||||
|
_last_token_renewal_date = None
|
||||||
|
|
||||||
|
|
||||||
async def _poll_loop() -> None:
|
async def _poll_loop() -> None:
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
await _tick()
|
await _tick()
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("Scheduler tick failed")
|
logger.exception("Scheduler tick failed")
|
||||||
|
try:
|
||||||
|
await _token_renewal_tick()
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Token renewal tick failed")
|
||||||
await asyncio.sleep(_POLL_INTERVAL_SECONDS)
|
await asyncio.sleep(_POLL_INTERVAL_SECONDS)
|
||||||
|
|
||||||
|
|
||||||
|
async def _token_renewal_tick() -> None:
|
||||||
|
"""Once a day, renew any NetBird client-update API token nearing its
|
||||||
|
365-day expiry — keeps central update control working indefinitely
|
||||||
|
without anyone needing to notice or act.
|
||||||
|
"""
|
||||||
|
global _last_token_renewal_date
|
||||||
|
now = datetime.now()
|
||||||
|
if now.hour != _TOKEN_RENEW_CHECK_HOUR:
|
||||||
|
return
|
||||||
|
if _last_token_renewal_date == now.date():
|
||||||
|
return
|
||||||
|
_last_token_renewal_date = now.date()
|
||||||
|
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
cutoff = now - timedelta(days=_TOKEN_RENEW_AFTER_DAYS)
|
||||||
|
deployments = (
|
||||||
|
db.query(Deployment)
|
||||||
|
.filter(Deployment.netbird_api_token_encrypted.isnot(None))
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
due = [
|
||||||
|
d for d in deployments
|
||||||
|
if d.netbird_api_token_renewed_at is None or d.netbird_api_token_renewed_at < cutoff
|
||||||
|
]
|
||||||
|
if not due:
|
||||||
|
return
|
||||||
|
logger.info("Renewing NetBird API token for %d customer(s)...", len(due))
|
||||||
|
for d in due:
|
||||||
|
token = decrypt_value(d.netbird_api_token_encrypted)
|
||||||
|
result = await netbird_client_update_service.renew_token(d.container_prefix, token)
|
||||||
|
if result["ok"]:
|
||||||
|
d.netbird_api_token_encrypted = encrypt_value(result["token"])
|
||||||
|
d.netbird_api_token_renewed_at = now
|
||||||
|
db.commit()
|
||||||
|
logger.info("Renewed NetBird API token for %s.", d.container_prefix)
|
||||||
|
else:
|
||||||
|
logger.warning("Failed to renew NetBird API token for %s: %s", d.container_prefix, result.get("error"))
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
async def _tick() -> None:
|
async def _tick() -> None:
|
||||||
db = SessionLocal()
|
db = SessionLocal()
|
||||||
try:
|
try:
|
||||||
|
|||||||
Reference in New Issue
Block a user