api_server_fire_startup.py 3.5 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667
  1. """Startup gate for the Chronos fire webhook (``POST /api/cron/fire``).
  2. A backend that stops the guest on sleep wakes the agent FOR the fire, so the fire reaches the webhook in
  3. the first second of a cold boot. The api_server adapter listens before the gateway has published any
  4. adapter into ``runner.adapters`` (an adapter is registered only after its ``connect()`` returns), so a
  5. fire taken then ran its job with no live adapters and a relay-fronted platform failed with "has no live
  6. gateway transport" (the result was lost). Wait for the gateway to finish starting instead.
  7. """
  8. from __future__ import annotations
  9. import asyncio
  10. from contextlib import suppress
  11. from typing import Any, Optional
  12. from aiohttp import web
  13. # How long a fire may spend in the handler (token verify included) before a still-starting gateway is
  14. # refused as retryable. Kept under the dashboard forwarder's 10s timeout: past it NAS sees a 503 while
  15. # this handler still runs the job, and the retry would run it twice.
  16. FIRE_STARTUP_WAIT_SECONDS = 7.0
  17. _POLL_SECONDS = 0.1
  18. def runner_is_draining(runner: Any) -> bool:
  19. """Whether ``runner`` refuses new work (an in-process drain, or one an external supervisor started)."""
  20. return bool(getattr(runner, "_draining", False) or getattr(runner, "_external_drain_active", False))
  21. async def refuse_until_started(runner: Any, job_id: str, *, received_at: float) -> Optional[web.Response]:
  22. """None once ``runner`` has finished starting (``_running``: every platform connect attempted and its
  23. adapters published), or when there is no runner to ask (a self-hosted api_server, a test double).
  24. Otherwise the retryable 503, worded like the dashboard's own "gateway unreachable" so NAS classifies
  25. it the same, with the dashboard's 60s Retry-After (``_CRON_FIRE_RETRY_AFTER_SECONDS``).
  26. ``received_at`` is the loop time the handler was entered: time already spent verifying the token
  27. counts against the budget."""
  28. if runner is None:
  29. return None
  30. loop = asyncio.get_running_loop()
  31. deadline = received_at + FIRE_STARTUP_WAIT_SECONDS
  32. while True:
  33. # Re-checked after every poll: the handler's drain check ran before this wait, and a gateway that
  34. # starts draining meanwhile (a restart keeps _running True; a stop mid-boot never sets it) must not
  35. # claim the fire.
  36. draining = runner_is_draining(runner)
  37. if not draining and getattr(runner, "_running", True):
  38. return None
  39. if draining or loop.time() >= deadline:
  40. return web.json_response(
  41. {"error": "gateway unreachable; retry", "job_id": job_id}, status=503, headers={"Retry-After": "60"})
  42. await asyncio.sleep(_POLL_SECONDS)
  43. async def live_adapters_once_started(
  44. api_adapter: Any, request: web.Request, job_id: str, *, received_at: float,
  45. ) -> tuple[Optional[web.Response], Any]:
  46. """``(refusal, adapters)`` for a fire: the startup gate's retryable 503, else the gateway's LIVE adapters
  47. (parity with the built-in ticker: E2EE / relay-fronted platforms have no native credential, so without
  48. them delivery fails). ``None`` adapters when there is no runner (a self-hosted api_server)."""
  49. runner = api_adapter.gateway_runner or request.app.get("gateway_runner")
  50. if runner is None:
  51. with suppress(Exception):
  52. from gateway.run import _gateway_runner_ref
  53. runner = _gateway_runner_ref()
  54. refusal = await refuse_until_started(runner, job_id, received_at=received_at)
  55. return refusal, (getattr(runner, "adapters", None) or None)