twothatITandClaude Sonnet 5
189c49d505
Some customers only regain access to their NetBird web UI after a manual container restart, and until now that required someone to notice and do it by hand. The scheduler now optionally probes each customer's actual dashboard URL over HTTP(S) every poll interval (not just Docker container state, since a container can be "running" while the web UI behind it still doesn't respond) and restarts that customer's containers if it doesn't answer, gated by a per-deployment cooldown to avoid restart loops on a persistently broken customer. New system setting "Automatic Restart of Unreachable Containers" (default off) with a configurable cooldown. Every automatic restart is logged in the affected customer's own log, same as a manual restart. Co-Authored-By: Claude Sonnet 5 <[email protected]>
284 lines
11 KiB
Python
284 lines
11 KiB
Python
"""Background scheduler for automatic NetBird image update checks.
|
|
|
|
No external scheduler dependency (APScheduler etc.) — a single asyncio task
|
|
started at app startup wakes up once a minute, and only actually does
|
|
anything once per day at the configured HH:MM, controlled entirely by
|
|
SystemConfig.auto_update_check_enabled / auto_update_check_time.
|
|
"""
|
|
|
|
import asyncio
|
|
import logging
|
|
from datetime import datetime, timedelta
|
|
|
|
from app.database import SessionLocal
|
|
from app.models import Deployment, SystemConfig
|
|
from app.services import (
|
|
docker_service,
|
|
image_service,
|
|
netbird_client_update_service,
|
|
netbird_service,
|
|
update_log_service,
|
|
)
|
|
from app.services.netbird_service import _log_action
|
|
from app.utils.security import decrypt_value, encrypt_value
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_POLL_INTERVAL_SECONDS = 60
|
|
_task: asyncio.Task | None = None
|
|
|
|
# NetBird PATs we mint are issued for 365 days; renew well before that so a
|
|
# missed tick or a slow rollout never risks the token actually expiring.
|
|
_TOKEN_RENEW_AFTER_DAYS = 300
|
|
_TOKEN_RENEW_CHECK_HOUR = 4 # run once per day, distinct from the image-check hour
|
|
|
|
|
|
def start() -> None:
|
|
"""Start the background polling task. Safe to call once at app startup."""
|
|
global _task
|
|
if _task is None or _task.done():
|
|
_task = asyncio.create_task(_poll_loop())
|
|
logger.info("Automatic update scheduler started.")
|
|
|
|
|
|
def stop() -> None:
|
|
"""Cancel the background polling task."""
|
|
global _task
|
|
if _task is not None:
|
|
_task.cancel()
|
|
_task = None
|
|
|
|
|
|
_last_token_renewal_date = None
|
|
|
|
|
|
async def _poll_loop() -> None:
|
|
while True:
|
|
try:
|
|
await _tick()
|
|
except Exception:
|
|
logger.exception("Scheduler tick failed")
|
|
try:
|
|
await _token_renewal_tick()
|
|
except Exception:
|
|
logger.exception("Token renewal tick failed")
|
|
try:
|
|
await _auto_restart_tick()
|
|
except Exception:
|
|
logger.exception("Auto-restart tick failed")
|
|
await asyncio.sleep(_POLL_INTERVAL_SECONDS)
|
|
|
|
|
|
async def _token_renewal_tick() -> None:
|
|
"""Once a day, renew any NetBird client-update API token nearing its
|
|
365-day expiry — keeps central update control working indefinitely
|
|
without anyone needing to notice or act.
|
|
"""
|
|
global _last_token_renewal_date
|
|
now = datetime.now()
|
|
if now.hour != _TOKEN_RENEW_CHECK_HOUR:
|
|
return
|
|
if _last_token_renewal_date == now.date():
|
|
return
|
|
_last_token_renewal_date = now.date()
|
|
|
|
db = SessionLocal()
|
|
try:
|
|
cutoff = now - timedelta(days=_TOKEN_RENEW_AFTER_DAYS)
|
|
deployments = (
|
|
db.query(Deployment)
|
|
.filter(Deployment.netbird_api_token_encrypted.isnot(None))
|
|
.all()
|
|
)
|
|
due = [
|
|
d for d in deployments
|
|
if d.netbird_api_token_renewed_at is None or d.netbird_api_token_renewed_at < cutoff
|
|
]
|
|
if not due:
|
|
return
|
|
logger.info("Renewing NetBird API token for %d customer(s)...", len(due))
|
|
for d in due:
|
|
token = decrypt_value(d.netbird_api_token_encrypted)
|
|
result = await netbird_client_update_service.renew_token(d.container_prefix, token)
|
|
if result["ok"]:
|
|
d.netbird_api_token_encrypted = encrypt_value(result["token"])
|
|
d.netbird_api_token_renewed_at = now
|
|
db.commit()
|
|
logger.info("Renewed NetBird API token for %s.", d.container_prefix)
|
|
else:
|
|
logger.warning("Failed to renew NetBird API token for %s: %s", d.container_prefix, result.get("error"))
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
async def _tick() -> None:
|
|
db = SessionLocal()
|
|
try:
|
|
config = db.query(SystemConfig).filter(SystemConfig.id == 1).first()
|
|
if not config or not config.auto_update_check_enabled:
|
|
return
|
|
|
|
now = datetime.now()
|
|
target_time = config.auto_update_check_time or "03:00"
|
|
current_hhmm = now.strftime("%H:%M")
|
|
if current_hhmm != target_time:
|
|
return
|
|
|
|
last_run = config.auto_update_last_run_at
|
|
if last_run and last_run.date() == now.date():
|
|
return # already ran today
|
|
|
|
# Claim this run immediately so a slow run can't overlap the next tick.
|
|
config.auto_update_last_run_at = now
|
|
db.commit()
|
|
|
|
apply_enabled = bool(config.auto_update_apply_enabled)
|
|
logger.info(
|
|
"Running scheduled NetBird image update check (auto-apply=%s)...", apply_enabled
|
|
)
|
|
await _run_check_and_optionally_apply(config, apply_enabled, started_at=now)
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
async def _run_check_and_optionally_apply(config: SystemConfig, apply_enabled: bool, started_at: datetime) -> None:
|
|
"""Check Docker Hub, pull if needed, then (if enabled) recreate any
|
|
customer container that isn't running the locally-pulled image yet.
|
|
|
|
The "recreate" step is intentionally NOT gated on a Hub update having
|
|
just been found: a customer can already be behind the *locally pulled*
|
|
image from an earlier check/pull that was never applied (e.g. auto-apply
|
|
was off at the time, or a previous run failed) — that backlog must keep
|
|
being cleared on every run, not just on runs that also found something
|
|
new on Hub, otherwise it silently never happens.
|
|
"""
|
|
log_db = SessionLocal()
|
|
try:
|
|
hub_status = await image_service.check_all_images(config)
|
|
any_update = hub_status["any_update_available"]
|
|
|
|
pull_attempted = False
|
|
pull_results = None
|
|
if any_update:
|
|
logger.info("Scheduled check: new NetBird image(s) available — pulling.")
|
|
pull_attempted = True
|
|
pull_result = await image_service.pull_all_images(config)
|
|
pull_results = pull_result["results"]
|
|
if not pull_result["all_success"]:
|
|
logger.error("Scheduled image pull had failures: %s", pull_results)
|
|
else:
|
|
logger.info("Scheduled check: all NetBird images already up to date on Docker Hub.")
|
|
|
|
if not apply_enabled:
|
|
logger.info("Auto-apply disabled — customer containers left untouched.")
|
|
update_log_service.record_run(
|
|
log_db, run_type="scheduled", trigger="scheduler",
|
|
started_at=started_at, any_update_available=any_update,
|
|
pull_attempted=pull_attempted, pull_results=pull_results,
|
|
)
|
|
return
|
|
|
|
deployments = log_db.query(Deployment).all()
|
|
to_update = []
|
|
for dep in deployments:
|
|
cs = image_service.get_customer_container_image_status(dep.container_prefix, config)
|
|
if cs["needs_update"]:
|
|
customer = dep.customer
|
|
to_update.append({
|
|
"instance_dir": f"{config.data_dir}/{customer.subdomain}",
|
|
"project_name": dep.container_prefix,
|
|
"customer_name": customer.name,
|
|
})
|
|
logger.info("Scheduled auto-apply: updating %d customer(s)...", len(to_update))
|
|
customer_results = []
|
|
for entry in to_update:
|
|
try:
|
|
res = await image_service.update_customer_containers(
|
|
entry["instance_dir"], entry["project_name"]
|
|
)
|
|
ok = res["success"]
|
|
err = res.get("error")
|
|
except Exception as exc:
|
|
logger.exception("Scheduled update failed for %s", entry["customer_name"])
|
|
ok = False
|
|
err = str(exc)
|
|
logger.info("Scheduled update for %s: %s", entry["customer_name"], "OK" if ok else err)
|
|
customer_results.append({"customer_name": entry["customer_name"], "success": ok, "error": err})
|
|
|
|
update_log_service.record_run(
|
|
log_db, run_type="scheduled", trigger="scheduler",
|
|
started_at=started_at, any_update_available=any_update,
|
|
pull_attempted=pull_attempted, pull_results=pull_results,
|
|
apply_attempted=True, customer_results=customer_results,
|
|
)
|
|
except Exception as exc:
|
|
logger.exception("Scheduled update run failed")
|
|
update_log_service.record_run(
|
|
log_db, run_type="scheduled", trigger="scheduler",
|
|
started_at=started_at, error=str(exc),
|
|
)
|
|
finally:
|
|
log_db.close()
|
|
|
|
|
|
async def _auto_restart_tick() -> None:
|
|
"""Every poll interval, restart any customer whose NetBird dashboard
|
|
doesn't actually respond over HTTP(S) — a container can be "running" in
|
|
Docker while the web UI behind it is still unreachable, which is the
|
|
actual symptom customers hit, so this probes the real endpoint rather
|
|
than trusting container state alone. Falls back to container status
|
|
only if a customer has no setup_url yet. Guarded by
|
|
SystemConfig.auto_restart_unreachable_enabled and a per-deployment
|
|
cooldown so a persistently broken customer doesn't get restarted in a
|
|
tight loop, and skipped for deployments that aren't running/failed
|
|
(e.g. intentionally stopped or still pending).
|
|
"""
|
|
db = SessionLocal()
|
|
try:
|
|
config = db.query(SystemConfig).filter(SystemConfig.id == 1).first()
|
|
if not config or not config.auto_restart_unreachable_enabled:
|
|
return
|
|
|
|
cooldown = timedelta(minutes=config.auto_restart_cooldown_minutes or 30)
|
|
now = datetime.now()
|
|
|
|
deployments = db.query(Deployment).all()
|
|
for dep in deployments:
|
|
if dep.deployment_status not in ("running", "failed"):
|
|
continue
|
|
if dep.last_auto_restart_at and now - dep.last_auto_restart_at < cooldown:
|
|
continue
|
|
|
|
reason = None
|
|
if dep.setup_url:
|
|
if not await netbird_service.check_dashboard_reachable(dep.setup_url):
|
|
reason = f"dashboard ({dep.setup_url}) did not respond"
|
|
else:
|
|
containers = await docker_service.get_container_status_async(dep.container_prefix)
|
|
if not containers or not all(c["status"] == "running" for c in containers):
|
|
not_running = [c["name"] for c in containers if c["status"] != "running"] or ["no containers found"]
|
|
reason = f"container(s) not running: {', '.join(not_running)}"
|
|
|
|
if not reason:
|
|
continue
|
|
|
|
logger.warning(
|
|
"Auto-restart: customer %s unreachable (%s) — restarting.",
|
|
dep.container_prefix, reason,
|
|
)
|
|
_log_action(
|
|
db, dep.customer_id, "auto_restart", "info",
|
|
f"Automatic restart triggered: {reason}.",
|
|
)
|
|
try:
|
|
result = await netbird_service.restart_customer(db, dep.customer_id)
|
|
if not result["success"]:
|
|
logger.error("Auto-restart failed for customer %s.", dep.container_prefix)
|
|
except Exception:
|
|
logger.exception("Auto-restart failed for customer %s.", dep.container_prefix)
|
|
|
|
dep.last_auto_restart_at = now
|
|
db.commit()
|
|
finally:
|
|
db.close()
|