scheduler_ownership.py 6.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165
  1. """Which profile homes THIS process ticks, and who owns the cron tick for a given home.
  2. One gateway process per host multiplexes every profile, so "does a gateway own cron for profile
  3. X" stopped being answerable from the launch home alone: ``gateway.status.owns_gateway_runtime_lock``
  4. is a process-global boolean set once at boot, and every path resolver behind
  5. ``is_gateway_runtime_lock_active`` goes through ``get_process_hermes_home()``, which ignores the
  6. per-tick ``_profile_cron_scope`` override. Both therefore answer about the LAUNCH home while the
  7. ticker is scoped to some other profile.
  8. The two predicates here keep those questions apart:
  9. * :func:`owns_cron_tick_for` — this process is the host gateway AND ticks that home.
  10. * :func:`live_gateway_ticking` — a DIFFERENT, live host gateway holds the runtime lock and its
  11. published served set covers that home.
  12. Every probe failure answers "no": ownership claims are certainties, never guesses.
  13. """
  14. from __future__ import annotations
  15. import threading
  16. from pathlib import Path
  17. from typing import Any, Dict, Optional, Union
  18. from hermes_constants import get_hermes_home, hermes_home_key
  19. # Home key -> home path for every profile this process ticks, republished each ticker cycle so a
  20. # profile created or tombstoned mid-run is reflected without a restart.
  21. _ticked_homes: dict[str, Path] = {}
  22. # Home key -> resolved cron store, taken while the home exists: a departed profile's home may be
  23. # gone by the time its store state is dropped, and a symlinked cron/ resolves elsewhere.
  24. _ticked_stores: dict[str, str] = {}
  25. _ticked_lock = threading.Lock()
  26. def register_ticked_homes(homes) -> None:
  27. """Publish the set of profile homes this process's cron ticker owns this cycle.
  28. A home that leaves the set also loses its cron worker pool: pools are per home and otherwise
  29. live until process exit, so serving (or churning) many profiles leaks one ThreadPoolExecutor
  30. and its worker threads per home ever ticked.
  31. """
  32. from cron.store_health import forget_stores, store_key
  33. resolved, stores = {}, {}
  34. for home in homes:
  35. path = Path(home)
  36. key = hermes_home_key(path)
  37. resolved[key] = path
  38. stores[key] = store_key(path)
  39. with _ticked_lock:
  40. departed = set(_ticked_homes) - set(resolved)
  41. # A store another still-ticked profile reaches (cron/ symlinked to one place) keeps its state.
  42. departed_stores = {_ticked_stores[k] for k in departed if k in _ticked_stores} - set(stores.values())
  43. _ticked_homes.clear()
  44. _ticked_homes.update(resolved)
  45. _ticked_stores.clear()
  46. _ticked_stores.update(stores)
  47. if departed:
  48. # Late import: cron.scheduler imports this module.
  49. from cron.scheduler import discard_parallel_pools
  50. discard_parallel_pools(departed)
  51. forget_stores(departed_stores)
  52. def ticked_homes() -> dict:
  53. """Snapshot of ``home key -> home`` for the profiles this process ticks."""
  54. with _ticked_lock:
  55. return dict(_ticked_homes)
  56. def serves_profile(home: Optional[Path | str] = None) -> bool:
  57. """True when THIS process's cron ticker owns ``home`` (default: the active cron scope)."""
  58. key = hermes_home_key(home if home is not None else get_hermes_home())
  59. with _ticked_lock:
  60. return key in _ticked_homes
  61. def owns_cron_tick_for(home: Optional[Path | str] = None) -> bool:
  62. """True when this process is the host gateway multiplexer AND ticks ``home``.
  63. The runtime lock alone is not enough: it proves only that this process is the host gateway,
  64. not that its ticker visits the profile whose store is being ticked right now.
  65. """
  66. try:
  67. from gateway import status as gateway_status
  68. if not gateway_status.owns_gateway_runtime_lock():
  69. return False
  70. except Exception:
  71. return False
  72. return serves_profile(home)
  73. def record_serves_profile(record: Any, home: Optional[Path | str] = None) -> bool:
  74. """True when a gateway runtime-status ``record``'s served set covers ``home``.
  75. The record is read from the holder's own home, so it always proves that home — that is the
  76. pre-multiplex meaning, and a single-profile gateway publishes an EMPTY ``served_profiles``.
  77. Other profiles need an explicit entry in the published served set.
  78. """
  79. if not isinstance(record, dict):
  80. return False
  81. from gateway.status import (
  82. _get_process_hermes_home, _profile_label_for_home, _same_hermes_home)
  83. try:
  84. target = Path(home) if home is not None else get_hermes_home()
  85. record_home = record.get("hermes_home")
  86. if not isinstance(record_home, str) or not record_home.strip():
  87. record_home = _get_process_hermes_home()
  88. if _same_hermes_home(record_home, target):
  89. return True
  90. except Exception:
  91. return False
  92. served = record.get("served_profiles")
  93. if not isinstance(served, list):
  94. return False
  95. label = _profile_label_for_home(target)
  96. return label is not None and label in served
  97. def live_gateway_ticking(home: Optional[Path | str] = None) -> Optional[dict]:
  98. """Runtime-status record of ANOTHER live host gateway that ticks ``home``, else None.
  99. ``None`` covers every "we cannot prove it" case: this process holds the lock itself, no lock
  100. is held, the record does not belong to the lock holder, its heartbeat is stale, or its served
  101. set does not name ``home``.
  102. """
  103. try:
  104. from gateway import status as gateway_status
  105. if gateway_status.owns_gateway_runtime_lock():
  106. return None
  107. if not gateway_status.is_gateway_runtime_lock_active():
  108. return None
  109. holder_pid = gateway_status.get_running_pid(cleanup_stale=False)
  110. record = gateway_status.read_runtime_status()
  111. if (
  112. holder_pid is None
  113. or not isinstance(record, dict)
  114. or record.get("pid") != holder_pid
  115. or gateway_status.runtime_status_is_stale(record)
  116. ):
  117. return None
  118. except Exception:
  119. return None
  120. return record if record_serves_profile(record, home) else None
  121. def _claim_owner_is_dead(claim: dict[str, Any]) -> bool:
  122. """True when the claim's ``by`` names a process on THIS host that provably no longer exists.
  123. ``_machine_id()`` stamps ``host:pid[:token]``; a foreign host, an explicit HERMES_MACHINE_ID,
  124. or any liveness-probe failure returns False (fail safe: only a proven death shortens the TTL)."""
  125. parts = str(claim.get("by") or "").split(":")
  126. if len(parts) < 2 or not parts[1].isdigit():
  127. return False
  128. try:
  129. import socket
  130. if parts[0] != socket.gethostname():
  131. return False
  132. from gateway.status import _pid_exists
  133. return not _pid_exists(int(parts[1]))
  134. except Exception:
  135. return False