| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171 |
- """Physical worker liveness and cooperative Stop across every served authority."""
- import asyncio
- from gateway.session_authorities import all_authorities
- def executing_sessions(authority):
- for sid, live in getattr(authority, 'sessions', {}).items():
- execution = getattr(getattr(live, 'event_stream', None), 'execution', None) or {}
- task = live.task
- if execution.get('execution_generation') is not None and task is not None and not task.done():
- yield sid, live, execution['execution_generation']
- def uncounted_runtime_work(runner):
- """Count claims absent from the ordinary agent map, including pre-bootstrap managed turns."""
- running = getattr(runner, '_running_agents', {})
- authorities = all_authorities(runner)
- managed = sum(not worker.closed.is_set() for authority in authorities
- for worker in getattr(authority, '_managed_workers', {}).values())
- return (managed + sum(live.route not in running and sid not in getattr(authority, '_managed_workers', {})
- for authority in authorities for sid, live, _ in executing_sessions(authority))
- + sum(len(mutation_tasks(authority)) + len(side_agents(authority)) for authority in authorities))
- def side_agents(authority):
- """``(thread, holder)`` for each detached side agent (/bg, /btw, preview restart) whose thread
- has not unwound (``gateway/session_tui_side.py`` ``_spawn``). A stopped ``/btw`` whose fork
- is gone (``_SideQuestionStop.abandoned``) is no longer owner work."""
- from gateway.session_tui_side import _abandoned
- return [(thread, holder) for thread, holder in list(getattr(authority, '_side_agents', {}).items())
- if not _abandoned(holder)]
- def interrupt_side_agents(runner, reason):
- """Hard-interrupt every counted side agent's holder across served authorities (the stop
- drain waits for them, so its timed-out interrupt must reach them). Returns how many."""
- from agent.interrupt_compat import request_hard_interrupt
- count = 0
- for authority in all_authorities(runner):
- for _, holder in side_agents(authority):
- if holder and holder[0] is not None:
- count += bool(request_hard_interrupt(holder[0], reason))
- return count
- def persist_work_count(authority, _task=None):
- """Re-publish the runner's in-flight count when owner work it counts (a drain, a tracked
- mutation, a mailbox operation) ends: the agent-slot release that last wrote
- ``gateway_state.json`` ran while that work was still live, so without this write the
- persisted count never returns to zero."""
- persist = getattr(getattr(authority, 'runner', None), '_persist_active_agents', None)
- if persist is not None:
- persist()
- def mutation_tasks(authority):
- tasks = {*getattr(authority, '_mutation_tasks', ()), *getattr(authority, '_bot_mailbox_operations', ())}
- return [task for task in tasks if not task.done()]
- def track_mutation(authority, operation):
- tasks = getattr(authority, '_mutation_tasks', None)
- if tasks is None:
- tasks = authority._mutation_tasks = set()
- task = asyncio.create_task(operation)
- tasks.add(task)
- def finished(done):
- tasks.discard(done)
- if not done.cancelled():
- done.exception() # the observer may already have disconnected
- persist_work_count(authority)
- task.add_done_callback(finished)
- return task
- async def tracked_write(authority, write, then=None, *, ordered=False, after=None):
- """Run a synchronous ledger write (``write()``) in a worker thread, off the owner loop.
- A held SQLite writer (``_WRITE_PATIENCE_S``) must not freeze every session, timer, socket,
- Stop and approval. The write and its loop-side follow-up (``then(result)``, sync or a
- coroutine: publication, scheduling, the claim's execution stamp) are one task tracked like ``track_mutation``, so
- retirement joins it, and shielded, so a cancelled caller cannot separate a commit from it.
- ``ordered`` writes (admissions) commit in call order, as they did on the loop: worker threads
- alone would let a later input take an earlier FIFO position.
- ``after(result)`` (an admission's drain scheduling) runs in the CALLER's own step once the
- write returns, as it did when admission was synchronous: the caller registers its delivery
- waiter before any drain it scheduled can claim, settle or pause that admission. A caller
- cancelled while the commit is in flight hands ``after`` to the commit's completion instead,
- so a committed row is never stranded without a drain."""
- order = getattr(authority, '_ordered_writes', None)
- if ordered and order is None:
- order = authority._ordered_writes = asyncio.Lock()
- async def run():
- if ordered:
- async with order:
- result = await asyncio.to_thread(write)
- else:
- result = await asyncio.to_thread(write)
- if then is None:
- return result
- followed = then(result)
- return await followed if asyncio.iscoroutine(followed) else followed
- task = track_mutation(authority, run())
- if after is None:
- return await asyncio.shield(task)
- try:
- result = await asyncio.shield(task)
- except asyncio.CancelledError:
- def late(done):
- if not done.cancelled() and done.exception() is None:
- after(done.result())
- task.add_done_callback(late)
- raise
- after(result)
- return result
- def start_turn_worker(runner, worker, agent_holder, run_sync):
- authority = track_turn_worker(runner, worker, agent_holder)
- worker.executor_task = asyncio.ensure_future(runner._run_in_executor_with_context(run_sync))
- if authority is not None:
- # Normal settlement releases the retained agent promptly. A cancelled asyncio waiter is
- # not physical completion; leave that entry until worker_done proves the thread exited.
- worker.executor_task.add_done_callback(lambda _: list(physical_workers(authority)))
- def track_turn_worker(runner, worker, agent_holder):
- from gateway.session_authorities import active_authority
- authority = active_authority(runner)
- if authority is not None:
- workers = getattr(authority, '_turn_workers', None)
- if workers is None:
- workers = authority._turn_workers = {}
- list(physical_workers(authority))
- workers[id(worker)] = (worker, agent_holder)
- return authority
- def physical_workers(authority):
- workers = getattr(authority, '_turn_workers', {})
- for key, (worker, holder) in list(workers.items()):
- if worker.worker_done.is_set():
- del workers[key]
- else:
- yield worker, holder
- def stop_authority_work(authority):
- from gateway.run_runtime import stop_authority_turns
- from agent.interrupt_compat import request_hard_interrupt
- stop_authority_turns(authority, in_process=True)
- for _, holder in [*physical_workers(authority), *side_agents(authority)]:
- if holder and holder[0] is not None:
- request_hard_interrupt(holder[0], 'Profile runtime is stopping')
- for cancel in getattr(authority, '_cron_cancellations', {}).values():
- cancel.set()
- async def join_authority_work(authority, timeout):
- deadline = asyncio.get_running_loop().time() + timeout
- while True:
- stop_authority_work(authority)
- if (not list(physical_workers(authority)) and not list(executing_sessions(authority))
- and not mutation_tasks(authority) and not side_agents(authority)):
- return
- if asyncio.get_running_loop().time() >= deadline:
- raise TimeoutError('Profile workers did not stop; ownership retained')
- await asyncio.sleep(.05)
|