scheduler_worktree_maintenance.py 3.5 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283
  1. """Process-wide coordination for the cron tick's worktree prune (``cron.scheduler`` runs in here).
  2. One process ticks every served profile (``cron/AGENTS.md``), so the per-profile throttle in
  3. ``cron.scheduler`` legitimately lets each profile decide it is due, and each due profile gets its
  4. own ``cron-worktree-prune`` thread. Those threads used to fork ``git`` side by side, which starved a
  5. 338-profile gateway's event loop until the liveness watchdog exited 75 (#143255). Two guards:
  6. - **One sweep at a time.** The tick is sequential, the sweeps it dispatches are not; serialising
  7. them (discovery included) keeps the number of concurrent ``git`` children at one regardless of
  8. profile count. A waiting sweep holds no resources, so the wait is capped rather than unbounded.
  9. - **One prune per repo per interval.** Served profiles mostly resolve the same repos (the hermes
  10. checkout at least), and every prune pass ends in a footprint probe of the whole ``.worktrees/``,
  11. so the second profile to reach a shared repo within the interval has nothing to do.
  12. """
  13. import logging
  14. import os
  15. import threading
  16. import time
  17. from typing import Callable, Iterable, Optional
  18. logger = logging.getLogger(__name__)
  19. # Bounded so a wedged sweep cannot park a profile's thread forever; a normal sweep is one git pass
  20. # over a handful of repos.
  21. SWEEP_WAIT_SECONDS = 120.0
  22. _sweep_slot = threading.Semaphore(1)
  23. _last_worktree_prune_at: dict[str, float] = {} # canonical repo path -> monotonic stamp
  24. _lock = threading.Lock()
  25. def claim_sweep_slot(wait_seconds: Optional[float] = None) -> bool:
  26. """Take the process-wide prune slot, waiting for the sweep ahead of us (bounded)."""
  27. return _sweep_slot.acquire(timeout=SWEEP_WAIT_SECONDS if wait_seconds is None else wait_seconds)
  28. def release_sweep_slot() -> None:
  29. _sweep_slot.release()
  30. def claim_due_worktree_repos(repos: Iterable[str], interval_s: float) -> list[str]:
  31. """The subset of *repos* no served profile pruned within *interval_s*, stamped as claimed.
  32. Keyed on the path ``git`` resolves to, case-folded where the filesystem is (Windows).
  33. """
  34. now = time.monotonic()
  35. due = []
  36. with _lock:
  37. for repo in repos:
  38. key = os.path.normcase(os.path.realpath(repo))
  39. last = _last_worktree_prune_at.get(key)
  40. if last is None or now - last >= interval_s:
  41. _last_worktree_prune_at[key] = now
  42. due.append(repo)
  43. return due
  44. def run_sweep(discover: Callable[[], Iterable[str]], *, interval_seconds: float,
  45. wait_seconds: Optional[float] = None) -> None:
  46. """Prune the repos ``discover()`` reports that are due, from the cron tick's daemon thread.
  47. ``discover`` stays in the caller so a per-profile repo lookup (and anything a test patches) is
  48. read where it is defined. Errors never propagate: GC is hygiene, not scheduling.
  49. """
  50. if not claim_sweep_slot(wait_seconds):
  51. logger.debug("Cron worktree maintenance skipped: another sweep is still running")
  52. return
  53. try:
  54. repos = claim_due_worktree_repos(discover() or [], interval_seconds)
  55. if not repos:
  56. return
  57. from hermes_cli.worktree_ops import _prune_stale_worktrees
  58. for repo in repos:
  59. try:
  60. _prune_stale_worktrees(repo)
  61. except Exception:
  62. logger.debug("Cron worktree maintenance failed for %s", repo, exc_info=True)
  63. except Exception:
  64. logger.debug("Cron worktree maintenance skipped", exc_info=True)
  65. finally:
  66. release_sweep_slot()