session_worker_reservation.py 7.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134
  1. """Private owner-side reservation for a concrete already-started admission.
  2. The producer retains the Popen handle and holds the child's bootstrap pipe until
  3. this returns. Public worker.register remains idle-only for unreserved callers.
  4. """
  5. import os
  6. import secrets
  7. import subprocess
  8. from gateway.session_admission import admission_fingerprint
  9. from gateway.owner_identity import same_owner
  10. from hermes_state_runtime import RuntimeStoreError, _admission, _epoch, _secret_digest, _session
  11. # A venv launcher (uv's Windows python.exe, pip's exe shim) sits between the owner's
  12. # handle and the interpreter; the real worker is never further away than this.
  13. MAX_LAUNCHER_HOPS = 3
  14. def verify_worker_identity(process, hello):
  15. """Return the verified ``(pid, birth)`` of the interpreter behind ``process``.
  16. ``Popen.pid`` may be a launcher trampoline, so the worker reports its own pid and
  17. birth in a ``hello`` frame; the OWNER proves that pid is alive, carries that birth,
  18. and descends from (or is) the reserved handle. The worker never decides equality.
  19. """
  20. import psutil
  21. if not isinstance(process, subprocess.Popen) or process.poll() is not None:
  22. raise RuntimeStoreError('worker_not_live')
  23. if (not isinstance(hello, dict) or hello.get('type') != 'hello' or type(hello.get('pid')) is not int
  24. or type(hello.get('birth')) not in (int, float) or not isinstance(hello.get('ancestors'), list)
  25. or any(type(p) is not int for p in hello['ancestors'])):
  26. raise RuntimeStoreError('invalid_worker_frame')
  27. pid = hello['pid']
  28. if pid <= 0 or pid == os.getpid():
  29. raise RuntimeStoreError('permission_denied')
  30. try:
  31. handle = psutil.Process(process.pid)
  32. if handle.ppid() != os.getpid() or handle.status() == psutil.STATUS_ZOMBIE:
  33. raise RuntimeStoreError('permission_denied')
  34. worker = psutil.Process(pid)
  35. from hermes_cli.process_identity import worker_birth
  36. birth = worker_birth(worker) # kernel start: stable across clock steps (#140580) and btime re-anchors
  37. if birth != hello['birth'] or not worker.is_running() or worker.status() == psutil.STATUS_ZOMBIE:
  38. raise RuntimeStoreError('permission_denied')
  39. # psutil's parent() already refuses a recycled ppid (parent born after child).
  40. chain = [p.pid for p in worker.parents()[:MAX_LAUNCHER_HOPS]]
  41. except psutil.Error as exc:
  42. raise RuntimeStoreError('worker_not_live') from exc
  43. if pid != process.pid and process.pid not in chain:
  44. raise RuntimeStoreError('permission_denied')
  45. return pid, birth
  46. def owner_worker_subject():
  47. """The subject a managed worker's adoption ticket carries: the owner's own OS identity. The
  48. control channel mints a worker ticket only for a same-user peer (``uid:<n>`` / ``sid:<sid>``)."""
  49. if os.name == 'nt':
  50. from gateway.runtime_bootstrap_windows import _process_sid
  51. return 'sid:' + _process_sid(os.getpid())
  52. return f'uid:{os.getuid()}' # windows-footgun: ok — POSIX branch
  53. def reserve_admission_worker(authority, *, admission_id, process, principal_id, hello, worker_subject=None):
  54. """Return a private bootstrap scope; never accept caller-selected kind/target.
  55. Only the owner process may reserve its live direct exec child. The admission
  56. supplies physical session and generation; its durable principal must match
  57. the producer. Do not publish the returned secret on any client event stream.
  58. The adoption claim binds ``worker_subject``, the identity the worker will authenticate
  59. as (``owner_worker_subject``), when given. An automation admission's principal is a route
  60. (``automation:<session key>``), never a ticket subject, so a claim bound to it refused every
  61. adopt and each bot DM or completion wake on a managed session ended interrupted (#144139).
  62. """
  63. authority._require_admission_open()
  64. pid, birth = verify_worker_identity(process, hello)
  65. secret = secrets.token_urlsafe(32)
  66. def write(conn):
  67. _epoch(conn, authority.epoch)
  68. admission = _admission(conn, admission_id)
  69. if admission['status'] != 'started' or admission['owner_epoch'] != authority.epoch:
  70. raise RuntimeStoreError('producer_not_started')
  71. if not same_owner(admission['principal_id'], principal_id):
  72. raise RuntimeStoreError('permission_denied')
  73. owner, generation = admission['target_session_id'], admission['generation']
  74. if _session(conn, owner)['runtime_generation'] != generation:
  75. raise RuntimeStoreError('stale_generation')
  76. from hermes_state_local_lineage import local_physical_target
  77. sid = local_physical_target(conn, owner)
  78. execution_id = 'admission-worker:' + admission_id
  79. if conn.execute('SELECT 1 FROM worker_executions WHERE execution_id=?', (execution_id,)).fetchone():
  80. raise RuntimeStoreError('admission_conflict')
  81. if conn.execute("SELECT 1 FROM worker_executions WHERE session_id=? AND status!='terminal'", (sid,)).fetchone():
  82. raise RuntimeStoreError('stale_generation')
  83. # The private worker fence follows its physical assignment; admission/FIFO
  84. # generation and identity remain on the logical owner.
  85. conn.execute('UPDATE sessions SET runtime_generation=? WHERE id=?', (generation, sid))
  86. if sid != owner:
  87. conn.execute("UPDATE session_admissions SET lineage_json=json_insert(lineage_json,'$[#]',?) "
  88. "WHERE admission_id=?", (sid, admission_id))
  89. scope = dict(profile_id=authority.profile_id, session_id=sid, execution_id=execution_id,
  90. generation=generation, pid=pid, birth=birth, secret=secret)
  91. claim = admission_fingerprint(canonical_target=sid, payload=scope | {'principal': worker_subject or principal_id})
  92. conn.execute("INSERT INTO worker_executions(execution_id,session_id,kind,owner_epoch,generation,status,adoption_digest) "
  93. "VALUES(?,?,'compute',?,?,'registered',?)",
  94. (execution_id, sid, authority.epoch, generation, _secret_digest(claim)))
  95. return scope | {'epoch': authority.epoch}
  96. return authority.db._execute_write(write)
  97. def lose_admission_worker(authority, row, scope):
  98. """Revoke a private assignment and pause its FIFO; no inference replay.
  99. This also fences a result already in the pipe at loss/shutdown. Retention
  100. requires a still-started admission even when the generation has not changed.
  101. """
  102. def write(conn):
  103. _epoch(conn, scope['epoch'])
  104. current = _admission(conn, row['admission_id'])
  105. if (current['status'] != 'started' or current['owner_epoch'] != scope['epoch']
  106. or current['generation'] != scope['generation']):
  107. raise RuntimeStoreError('stale_generation')
  108. worker = conn.execute('SELECT * FROM worker_executions WHERE execution_id=?',
  109. (scope['execution_id'],)).fetchone()
  110. if worker is None or worker['generation'] != scope['generation'] or worker['owner_epoch'] != scope['epoch']:
  111. raise RuntimeStoreError('stale_generation')
  112. conn.execute("UPDATE worker_executions SET status='terminal' WHERE execution_id=?", (scope['execution_id'],))
  113. from hermes_state_runtime_workers import compact_terminal_receipts
  114. compact_terminal_receipts(conn, scope['execution_id'])
  115. conn.execute("UPDATE session_admissions SET status='unknown' WHERE admission_id=?", (row['admission_id'],))
  116. conn.execute('UPDATE sessions SET runtime_revision=runtime_revision+1 WHERE id=?', (current['target_session_id'],))
  117. authority.db._execute_write(write)