"""Cron: teardown of a worker that outlived its ``run_job``. ``ThreadPoolExecutor.shutdown(wait=False)`` after an inactivity timeout does not stop a worker already inside ``run_conversation``. Finalizing its SessionDB from ``run_job``'s ``finally`` would close a handle the worker is still writing to — the checkpoint/WAL-unlink overlap behind #102827. The worker's Future owns the teardown instead. """ from __future__ import annotations import concurrent.futures import contextlib import subprocess import threading from typing import Optional def _close_late_session_db_result(future: concurrent.futures.Future) -> None: """Done-callback: close a SessionDB whose constructor finished after run_job's init timeout (worker abandoned via ``shutdown(wait=False)``), else its .db/WAL/SHM handles leak to EMFILE. If the constructor later completes inside that abandoned worker, the Future's result — an open SessionDB holding .db / WAL / SHM file handles — would be orphaned and never closed, leaking descriptors until EMFILE (#72782). This callback retrieves and closes that eventual late result. """ with contextlib.suppress(Exception): db = future.result() if db is not None: from hermes_state_registry import release_or_close release_or_close(db) def defer_teardown_to_running_worker( future: Optional[concurrent.futures.Future], session_db, agent, job_id: str, job_name: str, cron_session_id: str, workdir: Optional[str] = None, ) -> bool: """Return True when the worker is still running and its Future will finalize the session and tear the agent down on completion; False when the caller must do it now.""" if future is None or future.done(): return False from cron.scheduler import _finalize_cron_session, _teardown_cron_agent def _finish(_future) -> None: try: if session_db: _finalize_cron_session(session_db, agent, job_id, job_name, cron_session_id, workdir=workdir) finally: _teardown_cron_agent(agent, job_id) # Runs inline if the worker finished between done() and here — still exactly once. future.add_done_callback(_finish) return True def reap_terminal_worker_in_background(process: subprocess.Popen) -> None: """Keep the reap contract when the waiter returns before the worker exits. The ledger turning terminal lets ``_wait_for_external_cron_worker_body`` return while the worker is still in final teardown. The gateway remains the worker's parent, so if nobody calls ``wait()`` afterwards the worker lingers as a zombie (STAT=Z) under the gateway until it is restarted (#114509). A short-lived daemon thread holds that single responsibility and ends with the process exit it waits for. """ threading.Thread( target=process.wait, name=f"cron-worker-reap-{getattr(process, 'pid', '?')}", daemon=True, ).start()