fix(monitoring): repair silent-false-positive update badge + auto-update scheduling
Customer container status checks looked up containers by an exact expected name. When a docker compose recreate got interrupted (e.g. a hung command previously killed the whole update-all batch on timeout), Compose could leave the old container renamed with a random hash prefix instead of removed. The exact-name lookup then found nothing, returned None, and that silently counted as "up to date" (green "Aktuell") instead of surfacing as unknown — affecting 5 customers on the appliance whose containers were actually still running under orphaned names. - _run_cmd no longer raises on subprocess timeout, so one stuck customer can't abort the rest of a batch update - repair_container_naming() self-heals orphaned hash-renamed containers by renaming them back before every status check and before recreate - update-all loop now catches per-customer exceptions instead of aborting - status responses expose "unknown" separately from "needs_update" so the UI shows a distinct grey badge instead of a false-positive green one - new settings: automatic daily update check (on/off + time), with an independent toggle for whether it also auto-recreates customer containers
This commit is contained in:
@@ -159,6 +159,7 @@ async def check_image_updates(
|
||||
"subdomain": customer.subdomain,
|
||||
"container_prefix": dep.container_prefix,
|
||||
"needs_update": cs["needs_update"],
|
||||
"unknown": cs.get("unknown", False),
|
||||
"services": cs["services"],
|
||||
})
|
||||
|
||||
@@ -231,7 +232,7 @@ async def customers_local_update_status(
|
||||
|
||||
async def _check(dep: Deployment) -> dict[str, Any]:
|
||||
cs = await image_service.get_customer_container_image_status_async(dep.container_prefix, config)
|
||||
return {"customer_id": dep.customer_id, "needs_update": cs["needs_update"]}
|
||||
return {"customer_id": dep.customer_id, "needs_update": cs["needs_update"], "unknown": cs.get("unknown", False)}
|
||||
|
||||
results = await asyncio.gather(*[_check(dep) for dep in deployments])
|
||||
results = list(results)
|
||||
@@ -274,19 +275,27 @@ async def update_all_customers(
|
||||
if not to_update:
|
||||
return {"message": "All customers are already up to date.", "updated": 0, "results": []}
|
||||
|
||||
# Update customers sequentially — one at a time
|
||||
# Update customers sequentially — one at a time. A failure for one
|
||||
# customer (e.g. a hung docker compose call) must not abort the rest of
|
||||
# the batch, otherwise later customers silently never get updated.
|
||||
update_results = []
|
||||
for entry in to_update:
|
||||
res = await image_service.update_customer_containers(
|
||||
entry["instance_dir"], entry["project_name"]
|
||||
)
|
||||
ok = res["success"]
|
||||
logger.info("Updated %s: %s", entry["project_name"], "OK" if ok else res.get("error"))
|
||||
try:
|
||||
res = await image_service.update_customer_containers(
|
||||
entry["instance_dir"], entry["project_name"]
|
||||
)
|
||||
ok = res["success"]
|
||||
error = res.get("error")
|
||||
except Exception as exc:
|
||||
logger.exception("Unexpected error updating %s", entry["project_name"])
|
||||
ok = False
|
||||
error = str(exc)
|
||||
logger.info("Updated %s: %s", entry["project_name"], "OK" if ok else error)
|
||||
update_results.append({
|
||||
"customer_name": entry["customer_name"],
|
||||
"customer_id": entry["customer_id"],
|
||||
"success": ok,
|
||||
"error": res.get("error"),
|
||||
"error": error,
|
||||
})
|
||||
|
||||
success_count = sum(1 for r in update_results if r["success"])
|
||||
|
||||
Reference in New Issue
Block a user