scheduler_detached_worker.py 3.0 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970
  1. """Cron: teardown of a worker that outlived its ``run_job``.
  2. ``ThreadPoolExecutor.shutdown(wait=False)`` after an inactivity timeout does not stop a
  3. worker already inside ``run_conversation``. Finalizing its SessionDB from ``run_job``'s
  4. ``finally`` would close a handle the worker is still writing to — the checkpoint/WAL-unlink
  5. overlap behind #102827. The worker's Future owns the teardown instead.
  6. """
  7. from __future__ import annotations
  8. import concurrent.futures
  9. import contextlib
  10. import subprocess
  11. import threading
  12. from typing import Optional
  13. def _close_late_session_db_result(future: concurrent.futures.Future) -> None:
  14. """Done-callback: close a SessionDB whose constructor finished after run_job's init timeout
  15. (worker abandoned via ``shutdown(wait=False)``), else its .db/WAL/SHM handles leak to EMFILE.
  16. If the constructor later completes inside that abandoned worker, the Future's result — an open
  17. SessionDB holding .db / WAL / SHM file handles — would be orphaned and never closed, leaking
  18. descriptors until EMFILE (#72782). This callback retrieves and closes that eventual late result.
  19. """
  20. with contextlib.suppress(Exception):
  21. db = future.result()
  22. if db is not None:
  23. from hermes_state_registry import release_or_close
  24. release_or_close(db)
  25. def defer_teardown_to_running_worker(
  26. future: Optional[concurrent.futures.Future], session_db, agent, job_id: str, job_name: str,
  27. cron_session_id: str, workdir: Optional[str] = None,
  28. ) -> bool:
  29. """Return True when the worker is still running and its Future will finalize the session
  30. and tear the agent down on completion; False when the caller must do it now."""
  31. if future is None or future.done():
  32. return False
  33. from cron.scheduler import _finalize_cron_session, _teardown_cron_agent
  34. def _finish(_future) -> None:
  35. try:
  36. if session_db:
  37. _finalize_cron_session(session_db, agent, job_id, job_name, cron_session_id,
  38. workdir=workdir)
  39. finally:
  40. _teardown_cron_agent(agent, job_id)
  41. # Runs inline if the worker finished between done() and here — still exactly once.
  42. future.add_done_callback(_finish)
  43. return True
  44. def reap_terminal_worker_in_background(process: subprocess.Popen) -> None:
  45. """Keep the reap contract when the waiter returns before the worker exits.
  46. The ledger turning terminal lets ``_wait_for_external_cron_worker_body``
  47. return while the worker is still in final teardown. The gateway remains the
  48. worker's parent, so if nobody calls ``wait()`` afterwards the worker lingers
  49. as a zombie (STAT=Z) under the gateway until it is restarted (#114509). A
  50. short-lived daemon thread holds that single responsibility and ends with
  51. the process exit it waits for.
  52. """
  53. threading.Thread(
  54. target=process.wait,
  55. name=f"cron-worker-reap-{getattr(process, 'pid', '?')}",
  56. daemon=True,
  57. ).start()