kanban_watchers_common.py 5.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150
  1. """Plumbing shared by the kanban notifier and dispatcher loops.
  2. Thread offload, board enumeration, live-config coercers and the singleton
  3. dispatcher lock live here so the notifier, dispatcher and mixin modules read
  4. them from one place.
  5. """
  6. from __future__ import annotations
  7. import asyncio
  8. import contextlib
  9. import logging
  10. from contextvars import Context
  11. from pathlib import Path
  12. from typing import Any, Callable, Optional
  13. # Keep the logger name run.py used so extracted log records are unchanged.
  14. logger = logging.getLogger("gateway.run")
  15. def _run_in_fresh_context(func: Callable[..., Any], /, *args: Any) -> Any:
  16. """Run *func* in an empty ``Context`` so request-local ContextVars stay behind.
  17. ``asyncio.to_thread`` copies the caller's context; a lingering
  18. ``delegate_task`` child marker would make ``write_txn`` false-trip for
  19. these process-owned writers. An empty Context keeps the DB guard intact
  20. for real children without exempting dispatcher writes.
  21. """
  22. return Context().run(func, *args)
  23. async def _to_thread_process_service(func: Callable[..., Any], /, *args: Any) -> Any:
  24. """Offload blocking process-service work without inheriting request ContextVars."""
  25. return await asyncio.to_thread(_run_in_fresh_context, func, *args)
  26. def _list_boards(kb: Any) -> list:
  27. """Enumerate live boards; fall back to the default board when listing fails."""
  28. try:
  29. return kb.list_boards(include_archived=False)
  30. except Exception:
  31. return [kb.read_board_metadata(kb.DEFAULT_BOARD)]
  32. def _board_slugs(kb: Any) -> list:
  33. return [b.get("slug") or kb.DEFAULT_BOARD for b in _list_boards(kb)]
  34. def _positive_int_setting(kanban_cfg: dict, key: str) -> Optional[int]:
  35. """Parse an optional ``kanban.<key>`` int cap; None when unset or invalid (< 1 is invalid)."""
  36. raw = kanban_cfg.get(key)
  37. if raw is None:
  38. return None
  39. try:
  40. value = int(raw)
  41. except (TypeError, ValueError):
  42. logger.warning("kanban dispatcher: invalid kanban.%s=%r; ignoring", key, raw)
  43. return None
  44. if value < 1:
  45. logger.warning("kanban dispatcher: kanban.%s=%r is below 1; ignoring", key, raw)
  46. return None
  47. logger.info("kanban dispatcher: %s=%d", key, value)
  48. return value
  49. def _resolve_auto_decompose_settings(load_config: Callable[[], Any]) -> tuple[bool, int]:
  50. """Live (enabled, per_tick) auto-decompose settings, re-read every dispatcher tick.
  51. Fails safe: a config read error returns ``(False, 3)`` rather than
  52. re-enabling a feature the user turned off. ``per_tick`` is clamped to ``>= 1``.
  53. Read fresh from config on every dispatcher tick (#49638) so that flipping ``kanban.auto_decompose:
  54. false`` to STOP runaway fan-out takes effect on the next tick instead of requiring a gateway restart.
  55. Auto-decompose is a safety toggle — a user who sees it create and launch tasks they didn't intend
  56. reaches for this flag to halt it, and a stale boot-captured value silently ignoring that change is the
  57. bug reported in #49638.
  58. """
  59. try:
  60. cfg = load_config()
  61. except Exception:
  62. return False, 3
  63. kcfg = cfg.get("kanban", {}) if isinstance(cfg, dict) else {}
  64. try:
  65. per_tick = int(kcfg.get("auto_decompose_per_tick", 3) or 3)
  66. except (TypeError, ValueError):
  67. per_tick = 3
  68. return bool(kcfg.get("auto_decompose", True)), max(per_tick, 1)
  69. def _gc_retention_days() -> int:
  70. """``kanban.done_sub_retention_days`` (default 30; 0 disables), re-read per sweep; fails safe to 30."""
  71. try:
  72. from hermes_cli.config import load_config
  73. return int(((load_config() or {}).get("kanban") or {}).get("done_sub_retention_days", 30))
  74. except Exception:
  75. return 30
  76. def _kanban_dispatch_allowed() -> bool:
  77. """False while the global emergency stop (`hermes pause`) is engaged.
  78. Checked every tick before spawning, so a pause applies on the next tick;
  79. in-flight workers are never touched. Fails open if estop is unimportable.
  80. """
  81. try:
  82. from agent.estop import check_paused
  83. except ImportError:
  84. return True
  85. return not check_paused("kanban", logger)
  86. def _acquire_singleton_lock(lock_path) -> tuple[Optional[object], str]:
  87. """Take the exclusive, non-blocking advisory lock for the sole dispatcher.
  88. Only one gateway machine-wide may run the embedded dispatcher: concurrent
  89. dispatchers double reclaim frequency and claim events, and with
  90. ``wal_autocheckpoint=0`` concurrent manual checkpoints can corrupt index
  91. pages. ``dispatch_in_gateway`` is the primary control; this is the backstop.
  92. Returns ``(handle, "held")`` (release via :func:`_release_singleton_lock`),
  93. ``(None, "contended")`` when another process holds it (caller must NOT
  94. dispatch), or ``(None, "unavailable")`` when locking cannot be performed
  95. (caller falls back to config control).
  96. """
  97. try:
  98. from gateway.status import _try_acquire_file_lock # deferred; same package
  99. except ImportError:
  100. return None, "unavailable"
  101. try:
  102. Path(lock_path).parent.mkdir(parents=True, exist_ok=True)
  103. handle = open(str(lock_path), "a+", encoding="utf-8") # windows-footgun: ok (append-mode lock handle, write not read)
  104. except OSError:
  105. return None, "unavailable"
  106. if not _try_acquire_file_lock(handle):
  107. handle.close()
  108. return None, "contended"
  109. return handle, "held"
  110. def _release_singleton_lock(handle) -> None:
  111. """Release a lock acquired via :func:`_acquire_singleton_lock`."""
  112. if handle is None:
  113. return
  114. with contextlib.suppress(Exception):
  115. from gateway.status import _release_file_lock
  116. _release_file_lock(handle)
  117. with contextlib.suppress(Exception):
  118. handle.close()