session_worker.py 5.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101
  1. """Execution-scoped worker operations on the ordinary authenticated authority.
  2. Only explicit local compute registrations are enabled here. A delegated child is reserved by
  3. the owner on its parent's receipt stream (gateway/session_worker_child.py), never registered.
  4. """
  5. import asyncio
  6. import hmac
  7. import os
  8. from gateway.session_admission import admission_fingerprint
  9. from hermes_state_runtime import (
  10. RuntimeStoreError, _secret_digest, adopt_worker_execution,
  11. mutate_worker_execution, register_worker_execution,
  12. )
  13. _SCOPE = {'profile_id', 'session_id', 'execution_id', 'generation', 'pid', 'birth', 'secret'}
  14. def _claim(connection, ref, params):
  15. import psutil
  16. authority = connection.authority
  17. if params.get('profile_id') != authority.profile_id or connection.actor.profile_id != authority.profile_id:
  18. raise RuntimeStoreError('profile_mismatch')
  19. if not connection.actor.capabilities:
  20. raise RuntimeStoreError('permission_denied')
  21. pid, birth = params.get('pid'), params.get('birth')
  22. secret = params.get('secret')
  23. if (type(pid) is not int or pid <= 0 or pid == os.getpid()
  24. or type(birth) not in (int, float) or not isinstance(secret, str) or not 16 <= len(secret) <= 256):
  25. raise RuntimeStoreError('invalid_params')
  26. try:
  27. process = psutil.Process(pid)
  28. from hermes_cli.process_identity import worker_birth
  29. if (worker_birth(process) != birth or not process.is_running()
  30. or process.status() == psutil.STATUS_ZOMBIE
  31. or process.username() != psutil.Process().username()):
  32. raise RuntimeStoreError('permission_denied')
  33. except psutil.Error as exc:
  34. raise RuntimeStoreError('worker_not_live') from exc
  35. # Reuse the durable adoption digest as the producer claim. Changing any
  36. # assignment, principal, profile, PID birth or secret cannot adopt it.
  37. from hermes_state_worker_compression import worker_claim_origin
  38. origin = worker_claim_origin(authority.db, params)
  39. return admission_fingerprint(canonical_target=origin, payload={
  40. key: params[key] for key in _SCOPE} | {'session_id': origin, 'principal': connection.actor.subject})
  41. def _verify(connection, ref, params, claim):
  42. from hermes_state_worker_compression import worker_retry_target
  43. authority = connection.authority
  44. target = worker_retry_target(authority.db, params)
  45. with authority.db._read_ctx() as conn:
  46. row = conn.execute('SELECT * FROM worker_executions WHERE execution_id=?',
  47. (params['execution_id'],)).fetchone()
  48. if row is None:
  49. raise RuntimeStoreError('not_found')
  50. if row['session_id'] != target:
  51. raise RuntimeStoreError('permission_denied')
  52. if type(params['generation']) is not int or row['generation'] != params['generation']:
  53. raise RuntimeStoreError('stale_generation')
  54. if not hmac.compare_digest(row['adoption_digest'], _secret_digest(claim)):
  55. raise RuntimeStoreError('permission_denied')
  56. async def worker_request(connection, ref, params, *, operation):
  57. from gateway.session_authorities import require_serving
  58. require_serving(connection.authority) # adopt/persist would write with the released epoch
  59. if operation == "persist" and isinstance(params.get("operation"), str) and params["operation"].startswith("delegation."):
  60. from gateway.session_worker_delegation import worker_delegation_request
  61. return await worker_delegation_request(connection, ref, params)
  62. if operation == "persist" and params.get("operation") == "child.reserve":
  63. from gateway.session_worker_child import worker_child_request
  64. return await worker_child_request(connection, ref, params)
  65. extra = {'register': {'kind'}, 'adopt': set(), 'persist': {'epoch', 'sequence', 'operation', 'payload'}}[operation]
  66. if set(params) != _SCOPE | extra:
  67. raise RuntimeStoreError('invalid_params')
  68. claim = _claim(connection, ref, params)
  69. authority = connection.authority
  70. if operation == 'register':
  71. authority._require_admission_open()
  72. scope = {key: params[key] for key in ('execution_id', 'session_id', 'generation')}
  73. if operation == 'register':
  74. # Only a worker-adoption ticket (what a launched worker carries): an interactive client
  75. # must not park a 'registered' row that nothing reaps (the FIFO would pause until restart).
  76. authority.authorize(connection.actor, ref, 'worker:adopt')
  77. if params['kind'] != 'compute':
  78. raise RuntimeStoreError('unsupported_producer')
  79. return await asyncio.to_thread(register_worker_execution, authority.db, epoch=authority.epoch,
  80. **scope, kind='compute', adoption_secret=claim, require_idle=True)
  81. if 'worker:adopt' not in connection.actor.capabilities:
  82. raise RuntimeStoreError('permission_denied')
  83. _verify(connection, ref, params, claim)
  84. if operation == 'adopt':
  85. return await asyncio.to_thread(adopt_worker_execution, authority.db, epoch=authority.epoch,
  86. **scope, adoption_secret=claim)
  87. result = await asyncio.to_thread(mutate_worker_execution, authority.db, epoch=params['epoch'],
  88. **scope, sequence=params['sequence'], operation=params['operation'], payload=params['payload'])
  89. if params['operation'] == 'execution.finish':
  90. # Queued input parked behind this worker; nothing else would restart its FIFO.
  91. authority.wake_after_worker(params['session_id'])
  92. return result