worker.py 4.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125
  1. """One stdlib JSON-line PM request per isolated process."""
  2. from __future__ import annotations
  3. import json
  4. import os
  5. from pathlib import Path
  6. import queue
  7. import sys
  8. import threading
  9. def _read_controls(messages, pause, fd):
  10. # Raw reads avoid a daemon thread holding sys.stdin's buffered lock at exit.
  11. pending = b""
  12. request_id = None
  13. try:
  14. while block := os.read(fd, 65536):
  15. pending += block
  16. while b"\n" in pending:
  17. line, pending = pending.split(b"\n", 1)
  18. message = json.loads(line)
  19. if request_id is None:
  20. request_id = message["id"]
  21. if message["id"] != request_id:
  22. raise ValueError("unexpected PM control request id")
  23. if message.get("cancel"):
  24. pause.set()
  25. if message.get("type") != "cancel":
  26. messages.put(message)
  27. except (OSError, ValueError, KeyError) as exc:
  28. messages.put(exc)
  29. finally:
  30. os.close(fd)
  31. pause.set()
  32. messages.put(None)
  33. def main():
  34. import truststore
  35. truststore.inject_into_ssl()
  36. # Capture the protocol FD before redirecting even native/subprocess stdout.
  37. wire = os.fdopen(os.dup(sys.stdout.fileno()), "w", encoding="utf-8", buffering=1)
  38. os.dup2(sys.stderr.fileno(), sys.stdout.fileno())
  39. sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
  40. # A pending read on inherited control stdin can block child Python startup
  41. # on Windows. Keep the protocol private and give every ordinary child EOF.
  42. controls = os.dup(0)
  43. os.set_inheritable(controls, False)
  44. with open(os.devnull, "rb") as null:
  45. os.dup2(null.fileno(), 0)
  46. messages = queue.Queue()
  47. pause = threading.Event()
  48. threading.Thread(target=_read_controls, args=(messages, pause, controls), daemon=True).start()
  49. def receive():
  50. message = messages.get()
  51. if message is None:
  52. raise RuntimeError("PM client disconnected")
  53. if isinstance(message, Exception):
  54. raise message
  55. return message
  56. request = receive()
  57. from pm import paths, plugin_inputs, receipt
  58. from pm.package import InstallError
  59. from pm.registry import load_package_definitions
  60. from pm.runtime import lease_current_runtime
  61. from pm.worker_operations import OPERATIONS
  62. lease_current_runtime()
  63. context = request["context"]
  64. paths.repo_root = lambda: Path(context["repo"])
  65. paths.lockfile_path = lambda: Path(context["lockfile"])
  66. call = 0
  67. callback_lock = threading.Lock()
  68. def send(data):
  69. wire.write(json.dumps({"id": request["id"], **data}) + "\n")
  70. def callback(name, *args):
  71. with callback_lock:
  72. return exchange(name, args)
  73. def exchange(name, args):
  74. nonlocal call
  75. call += 1
  76. send({"type": "callback", "callback": name, "call": call, "args": args})
  77. reply = receive()
  78. if reply["id"] != request["id"] or reply["call"] != call:
  79. raise RuntimeError("unexpected PM callback response")
  80. if "error" in reply:
  81. raise RuntimeError(reply["error"])
  82. return reply["result"]
  83. with receipt.worker_context(request.get("update_id")):
  84. try:
  85. load_package_definitions(request.get("packages", []))
  86. operation = request["operation"]
  87. implementation = OPERATIONS[operation].resolve(operation)
  88. arguments = request["arguments"]
  89. if request["operation"] in ("sync_venv", "venv_is_current"):
  90. arguments["plugins"] = plugin_inputs.decode(arguments.get("plugins"))
  91. if request["operation"] == "ensure":
  92. arguments["pause_event"] = pause
  93. for name in ("progress", "download_progress"):
  94. if name in request["callbacks"]:
  95. arguments[name] = lambda *args, name=name: callback(name, *args)
  96. result = implementation(**arguments)
  97. if request["operation"] == "ensure":
  98. result = None # Runner is reconstructed from the caller's base env.
  99. if isinstance(result, Path):
  100. result = str(result)
  101. response = {"result": result}
  102. except BaseException as exc:
  103. error = {"type": type(exc).__name__, "message": str(exc)}
  104. if isinstance(exc, InstallError):
  105. error.update(package=exc.package, cause=exc.cause, remedy=exc.remedy)
  106. response = {"error": error}
  107. response["receipt"] = receipt.last_completed()
  108. send({"type": "result", **response})
  109. if __name__ == "__main__":
  110. main()