cron_store_notices.py 5.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113
  1. """Home-channel notices for cron store degraded-state transitions (see ``cron/store_health.py``).
  2. The cron tick runs on a worker thread inside the gateway. When a profile's cron store becomes
  3. unwritable it reports ONE transition, and again when it recovers. Each one is posted to that
  4. profile's home channels, honouring that profile's warning-notification opt-out, through the same
  5. send path the state.db warning uses.
  6. """
  7. from __future__ import annotations
  8. import asyncio
  9. import contextlib
  10. import logging
  11. from pathlib import Path
  12. from agent.i18n import t
  13. logger = logging.getLogger(__name__)
  14. # A store flapping across the disk-full line must not post a notice pair every tick: "recovered"
  15. # waits this long and is dropped if the store degrades again meanwhile.
  16. NOTICE_REPEAT_SECONDS = 3600.0
  17. def install_cron_store_notices(runner, loop: asyncio.AbstractEventLoop) -> None:
  18. """Route store transitions from the ticker thread onto the gateway loop. Installed once the
  19. adapters are connected, so a store that degraded during boot is announced then. The last
  20. notice a home channel sees always matches the store's real state."""
  21. from cron.store_health import degraded_record, degraded_records, set_transition_listener
  22. # Loop-only state (mutated in loop callbacks, never on the ticker thread).
  23. announced: set[str] = set() # stores whose current outage got an "unwritable" notice
  24. pending: dict[str, asyncio.TimerHandle] = {} # store -> its deferred "recovered" notice
  25. sending: dict[str, asyncio.Lock] = {} # store -> serializes its notices, in transition order
  26. async def deliver(event, record) -> None:
  27. async with sending.setdefault(record.store, asyncio.Lock()):
  28. # Re-check at delivery: a send can wait on profile loading or its transport, and the
  29. # store may have flipped meanwhile; a stale notice must never be the last word.
  30. degraded = degraded_record(Path(record.store)) is not None
  31. if event == "unwritable" and not degraded: # never announced: nothing owed, next entry announces
  32. announced.discard(record.store)
  33. if (handle := pending.pop(record.store, None)) is not None:
  34. handle.cancel()
  35. return
  36. if event == "recovered":
  37. if degraded or record.store not in announced: # outage goes on / was never announced
  38. return
  39. announced.discard(record.store)
  40. await send_cron_store_notice(runner, event, record)
  41. def send(event, record) -> None:
  42. task = loop.create_task(deliver(event, record))
  43. task.add_done_callback(lambda done: _log_notice_failure(done, event, record.store))
  44. def send_recovered(record) -> None:
  45. pending.pop(record.store, None)
  46. if record.store in announced: # deliver() drops it if the store degraded again meanwhile
  47. send("recovered", record)
  48. def on_loop(event, record) -> None:
  49. store = record.store
  50. if event == "unwritable":
  51. if (handle := pending.pop(store, None)) is not None:
  52. handle.cancel() # never told it recovered: the outage notice still stands
  53. elif store not in announced: # deliver() drops a boot replay of a since-recovered store
  54. announced.add(store)
  55. send(event, record)
  56. elif store in announced and store not in pending: # no lone "recovered"
  57. pending[store] = loop.call_later(NOTICE_REPEAT_SECONDS, send_recovered, record)
  58. def on_transition(event, record) -> None: # the ticker thread
  59. # A closed loop raises before any coroutine exists; store_health._notify logs and swallows it.
  60. loop.call_soon_threadsafe(on_loop, event, record)
  61. set_transition_listener(on_transition)
  62. for record in degraded_records():
  63. on_transition("unwritable", record)
  64. def _log_notice_failure(done, event: str, store: str) -> None:
  65. try:
  66. done.result()
  67. except asyncio.CancelledError:
  68. return
  69. except Exception: # the ticker thread already moved on: log, never raise
  70. logger.warning("Cron store %s notice for %s failed", event, store, exc_info=True)
  71. async def send_cron_store_notice(runner, event: str, record) -> None:
  72. """Post one ``unwritable``/``recovered`` notice to the home channels of the store's profile."""
  73. from gateway.run import _async_profile_runtime_scope
  74. from gateway.warning_notifications import present_notification
  75. from cron.store_health import store_key
  76. from hermes_constants import get_routing_process_hermes_home
  77. fields = record.notice_fields()
  78. key = "gateway.cron_store.unwritable" if event == "unwritable" else "gateway.cron_store.recovered"
  79. served_homes = runner._served_profile_homes or {}
  80. logger.info("Broadcasting cron store %s notice for %s", event, record.store)
  81. for profile, platform, _cfg, home, transport in list(runner._served_home_channel_transports()):
  82. served_home = served_homes.get(profile) if profile is not None else None
  83. # record.store is resolved: a cron/ symlinked to another disk has a foreign parent.
  84. if store_key(served_home or get_routing_process_hermes_home()) != record.store:
  85. continue
  86. # The opt-out is the owning profile's; the launch profile needs no extra scope.
  87. scope = _async_profile_runtime_scope(Path(served_home)) if served_home else contextlib.nullcontext()
  88. async with scope:
  89. message = t(key, **fields) # inside the owning profile's scope: its display.language
  90. await present_notification(
  91. lambda: runner._send_home_channel_message(
  92. platform, home, transport, message, "Cron store notice failed for %s:%s: %s"),
  93. platform=platform)