session_runtime_workers.py 7.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171
  1. """Physical worker liveness and cooperative Stop across every served authority."""
  2. import asyncio
  3. from gateway.session_authorities import all_authorities
  4. def executing_sessions(authority):
  5. for sid, live in getattr(authority, 'sessions', {}).items():
  6. execution = getattr(getattr(live, 'event_stream', None), 'execution', None) or {}
  7. task = live.task
  8. if execution.get('execution_generation') is not None and task is not None and not task.done():
  9. yield sid, live, execution['execution_generation']
  10. def uncounted_runtime_work(runner):
  11. """Count claims absent from the ordinary agent map, including pre-bootstrap managed turns."""
  12. running = getattr(runner, '_running_agents', {})
  13. authorities = all_authorities(runner)
  14. managed = sum(not worker.closed.is_set() for authority in authorities
  15. for worker in getattr(authority, '_managed_workers', {}).values())
  16. return (managed + sum(live.route not in running and sid not in getattr(authority, '_managed_workers', {})
  17. for authority in authorities for sid, live, _ in executing_sessions(authority))
  18. + sum(len(mutation_tasks(authority)) + len(side_agents(authority)) for authority in authorities))
  19. def side_agents(authority):
  20. """``(thread, holder)`` for each detached side agent (/bg, /btw, preview restart) whose thread
  21. has not unwound (``gateway/session_tui_side.py`` ``_spawn``). A stopped ``/btw`` whose fork
  22. is gone (``_SideQuestionStop.abandoned``) is no longer owner work."""
  23. from gateway.session_tui_side import _abandoned
  24. return [(thread, holder) for thread, holder in list(getattr(authority, '_side_agents', {}).items())
  25. if not _abandoned(holder)]
  26. def interrupt_side_agents(runner, reason):
  27. """Hard-interrupt every counted side agent's holder across served authorities (the stop
  28. drain waits for them, so its timed-out interrupt must reach them). Returns how many."""
  29. from agent.interrupt_compat import request_hard_interrupt
  30. count = 0
  31. for authority in all_authorities(runner):
  32. for _, holder in side_agents(authority):
  33. if holder and holder[0] is not None:
  34. count += bool(request_hard_interrupt(holder[0], reason))
  35. return count
  36. def persist_work_count(authority, _task=None):
  37. """Re-publish the runner's in-flight count when owner work it counts (a drain, a tracked
  38. mutation, a mailbox operation) ends: the agent-slot release that last wrote
  39. ``gateway_state.json`` ran while that work was still live, so without this write the
  40. persisted count never returns to zero."""
  41. persist = getattr(getattr(authority, 'runner', None), '_persist_active_agents', None)
  42. if persist is not None:
  43. persist()
  44. def mutation_tasks(authority):
  45. tasks = {*getattr(authority, '_mutation_tasks', ()), *getattr(authority, '_bot_mailbox_operations', ())}
  46. return [task for task in tasks if not task.done()]
  47. def track_mutation(authority, operation):
  48. tasks = getattr(authority, '_mutation_tasks', None)
  49. if tasks is None:
  50. tasks = authority._mutation_tasks = set()
  51. task = asyncio.create_task(operation)
  52. tasks.add(task)
  53. def finished(done):
  54. tasks.discard(done)
  55. if not done.cancelled():
  56. done.exception() # the observer may already have disconnected
  57. persist_work_count(authority)
  58. task.add_done_callback(finished)
  59. return task
  60. async def tracked_write(authority, write, then=None, *, ordered=False, after=None):
  61. """Run a synchronous ledger write (``write()``) in a worker thread, off the owner loop.
  62. A held SQLite writer (``_WRITE_PATIENCE_S``) must not freeze every session, timer, socket,
  63. Stop and approval. The write and its loop-side follow-up (``then(result)``, sync or a
  64. coroutine: publication, scheduling, the claim's execution stamp) are one task tracked like ``track_mutation``, so
  65. retirement joins it, and shielded, so a cancelled caller cannot separate a commit from it.
  66. ``ordered`` writes (admissions) commit in call order, as they did on the loop: worker threads
  67. alone would let a later input take an earlier FIFO position.
  68. ``after(result)`` (an admission's drain scheduling) runs in the CALLER's own step once the
  69. write returns, as it did when admission was synchronous: the caller registers its delivery
  70. waiter before any drain it scheduled can claim, settle or pause that admission. A caller
  71. cancelled while the commit is in flight hands ``after`` to the commit's completion instead,
  72. so a committed row is never stranded without a drain."""
  73. order = getattr(authority, '_ordered_writes', None)
  74. if ordered and order is None:
  75. order = authority._ordered_writes = asyncio.Lock()
  76. async def run():
  77. if ordered:
  78. async with order:
  79. result = await asyncio.to_thread(write)
  80. else:
  81. result = await asyncio.to_thread(write)
  82. if then is None:
  83. return result
  84. followed = then(result)
  85. return await followed if asyncio.iscoroutine(followed) else followed
  86. task = track_mutation(authority, run())
  87. if after is None:
  88. return await asyncio.shield(task)
  89. try:
  90. result = await asyncio.shield(task)
  91. except asyncio.CancelledError:
  92. def late(done):
  93. if not done.cancelled() and done.exception() is None:
  94. after(done.result())
  95. task.add_done_callback(late)
  96. raise
  97. after(result)
  98. return result
  99. def start_turn_worker(runner, worker, agent_holder, run_sync):
  100. authority = track_turn_worker(runner, worker, agent_holder)
  101. worker.executor_task = asyncio.ensure_future(runner._run_in_executor_with_context(run_sync))
  102. if authority is not None:
  103. # Normal settlement releases the retained agent promptly. A cancelled asyncio waiter is
  104. # not physical completion; leave that entry until worker_done proves the thread exited.
  105. worker.executor_task.add_done_callback(lambda _: list(physical_workers(authority)))
  106. def track_turn_worker(runner, worker, agent_holder):
  107. from gateway.session_authorities import active_authority
  108. authority = active_authority(runner)
  109. if authority is not None:
  110. workers = getattr(authority, '_turn_workers', None)
  111. if workers is None:
  112. workers = authority._turn_workers = {}
  113. list(physical_workers(authority))
  114. workers[id(worker)] = (worker, agent_holder)
  115. return authority
  116. def physical_workers(authority):
  117. workers = getattr(authority, '_turn_workers', {})
  118. for key, (worker, holder) in list(workers.items()):
  119. if worker.worker_done.is_set():
  120. del workers[key]
  121. else:
  122. yield worker, holder
  123. def stop_authority_work(authority):
  124. from gateway.run_runtime import stop_authority_turns
  125. from agent.interrupt_compat import request_hard_interrupt
  126. stop_authority_turns(authority, in_process=True)
  127. for _, holder in [*physical_workers(authority), *side_agents(authority)]:
  128. if holder and holder[0] is not None:
  129. request_hard_interrupt(holder[0], 'Profile runtime is stopping')
  130. for cancel in getattr(authority, '_cron_cancellations', {}).values():
  131. cancel.set()
  132. async def join_authority_work(authority, timeout):
  133. deadline = asyncio.get_running_loop().time() + timeout
  134. while True:
  135. stop_authority_work(authority)
  136. if (not list(physical_workers(authority)) and not list(executing_sessions(authority))
  137. and not mutation_tasks(authority) and not side_agents(authority)):
  138. return
  139. if asyncio.get_running_loop().time() >= deadline:
  140. raise TimeoutError('Profile workers did not stop; ownership retained')
  141. await asyncio.sleep(.05)