status_inline_source.py 8.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162
  1. """Inline-source (``python -c <src> …``) command-line parsing for gateway process identity.
  2. Everything after ``-c`` is data for the inline program, not the process's own identity -- except
  3. Hermes' own bootstraps, which run an entry point in-process and are mapped back to the equivalent
  4. ``python -m <module> <argv…>`` (see #107002).
  5. """
  6. from __future__ import annotations
  7. import re
  8. def inline_source_flag_index(tokens: list[str]) -> int | None:
  9. """Index of the ``-c`` token when *tokens* is an interpreter running INLINE SOURCE, else None.
  10. Everything after ``-c`` is data the inline program receives, not this process's own identity.
  11. The detached gateway restart watcher (``gateway._spawn_gateway_restart_watcher``) is spawned as
  12. ``python -c <watcher source> <old_pid> <python> -m hermes_cli.main gateway run``: its trailing
  13. argv is the command the watcher will LATER spawn, so every argv matcher used to read it as a
  14. live gateway. See #107002 and the "never infer process identity from argv substrings" rule.
  15. Only interpreter options may precede ``-c``; the first non-option token ends the option block
  16. (``python -m hermes_cli.main …`` therefore never matches).
  17. The walk is VALUE-AWARE: ``-X``/``-W``/``-Q`` and ``--check-hash-based-pycs``/``--jit`` take a
  18. SEPARATE operand, so a naive "first non-option token ends the block" walk mistakes that operand
  19. for the end of the block and never reaches the ``-c`` behind it (``python -X utf8 -c <src> …``
  20. was still read as a live gateway). The operand sets are the canonical ones in
  21. ``hermes_state_holders``, not a second hand-rolled copy.
  22. *tokens* must be CASE-PRESERVING: the operand-taking ``-Q``/``-W``/``-X`` differ from the
  23. operand-less ``-q``/``-b``, so a lowercased argv would skip the token after a plain ``-q``.
  24. """
  25. from hermes_state_holders import (
  26. _PYTHON_LONG_OPTIONS_WITH_OPERANDS,
  27. _PYTHON_SHORT_OPTIONS_WITH_OPERANDS,
  28. )
  29. index = 1
  30. while index < len(tokens):
  31. token = tokens[index]
  32. if token == "--":
  33. return None
  34. if token in _PYTHON_LONG_OPTIONS_WITH_OPERANDS:
  35. index += 2 # the next token is this option's operand, not the end of the option block
  36. continue
  37. if token.startswith("--"):
  38. index += 1 # ``--opt=value`` and operand-less long options
  39. continue
  40. if not token.startswith("-") or token == "-":
  41. return None
  42. # Clustered short options (``-uc``, ``-IsB``). An operand-taking letter consumes the rest of
  43. # the cluster as its attached value, or the following token when the cluster ends there --
  44. # so ``-Xc`` is ``-X c``, NOT an inline-source ``-c``.
  45. cluster = token[1:]
  46. for position, letter in enumerate(cluster):
  47. if letter == "c":
  48. return index
  49. if letter in _PYTHON_SHORT_OPTIONS_WITH_OPERANDS:
  50. index += 1 if cluster[position + 1 :] else 2
  51. break
  52. else:
  53. index += 1
  54. return None
  55. def command_line_runs_inline_source(tokens: list[str]) -> bool:
  56. """True when *tokens* is an interpreter running INLINE SOURCE (``python -c <src> [args]``)."""
  57. return inline_source_flag_index(tokens) is not None
  58. # Hermes' own inline bootstraps hand control to a Hermes entry point IN this process, so the argv
  59. # they run with is this process's own identity; every other ``-c`` program keeps its trailing argv
  60. # as data (#107002). Each pattern is one emitted source shape, anchored at both ends so a program
  61. # merely CARRYING a bootstrap command line (the restart watcher's respawn argv) never matches.
  62. _Q = r"""['"]?"""
  63. _MAIN = rf"{_Q}__main__{_Q}"
  64. _RUN_MODULE = rf"runpy\.run_module\(\s*{_Q}(?P<target>[\w.]+){_Q}\s*,\s*run_name\s*=\s*{_MAIN}\s*,\s*alter_sys\s*=\s*True\s*\)"
  65. _BOOTSTRAPS = (
  66. # hermes_cli._launchers.runtime_command (store launcher, the Windows updater's relaunch)
  67. ("module", re.compile(rf"import os, sys, runpy;.*\b{_RUN_MODULE}", re.DOTALL)),
  68. # hermes_cli.venv_sync.relaunch_command: argv is assigned inside the source
  69. ("module", re.compile(rf"import sys, runpy; sys\.path\.insert\(.*\b{_RUN_MODULE}", re.DOTALL)),
  70. ("path", re.compile(
  71. rf"import sys, runpy; sys\.path\.insert\(.*\brunpy\.run_path\(\s*{_Q}(?P<target>[^'\"]+?){_Q}\s*,\s*run_name\s*=\s*{_MAIN}\s*\)",
  72. re.DOTALL)),
  73. # hermes_cli.venv_sync.relaunch_command re-entering a ``-c`` launcher: the launcher's source is
  74. # exec'd as a string literal, its argv assigned before it; resolved through the rows above/below
  75. ("exec", re.compile(r"import sys, runpy; sys\.path\.insert\(.*?;\s*exec\((?P<target>.+)\)", re.DOTALL)),
  76. # hermes_cli._launchers._launcher_script (the published POSIX shell / Windows .cmd launcher)
  77. ("entry", re.compile(r"import os, re, sys\s.*\bfrom\s+(?P<target>[\w.]+)\s+import\s+(?P<func>\w+)\b.*\bsys\.exit\(\s*(?P=func)\(\)\s*\)", re.DOTALL)),
  78. # scripts/build/launchers.py::posix_launcher (the bundled/Docker payload's bin/hermes). Spelled out
  79. # statement by statement: a ``.*`` would span a carrier's source plus the launcher argv it carries.
  80. # Only the ``hermes`` entry: the same shape importing any other module is someone else's program.
  81. ("module", re.compile(
  82. rf"import os, site, sys;\s*sys\.argv\[0\]\s*=\s*{_Q}[\w.-]+{_Q};\s*"
  83. rf"site\.addsitedir\(os\.environ\[{_Q}HERMES_SITE{_Q}\]\);"
  84. r"\s*from\s+(?P<target>hermes_cli\.main)\s+import\s+main\s*;\s*sys\.exit\(\s*main\(\)\s*\)")),
  85. # hermes_cli._launchers._write_cmd_launcher: the launcher script, base64-encoded
  86. ("base64", re.compile(rf"import base64; exec\(base64\.b64decode\({_Q}(?P<target>[A-Za-z0-9+/=]+){_Q}\)\)")),
  87. )
  88. _ASSIGNED_ARGV = re.compile(r"\bsys\.argv\s*=\s*\[(.*?)\]\s*;")
  89. _EXEC_LAUNCHER = re.compile(
  90. # no ``\b`` before ``from``/``sys``: an escaped newline normalizes to ``/n`` and abuts them
  91. r"import os, re, sys\W.*?from\s+(?P<target>[\w.]+)\s+import\s+(?P<func>\w+)\b.*sys\.exit\(\s*(?P=func)\(\)\s*\)", re.DOTALL)
  92. _EXEC_BASE64 = re.compile(r"import base64; exec\(base64\.b64decode\(\W*(?P<b64>[A-Za-z0-9+/=]+)")
  93. def _bootstrap_entry(source: str, argv: list[str]) -> list[str] | None:
  94. """``[-m, <module>, *argv]`` (or ``[<path>, *argv]``) the inline *source* runs in-process, else None."""
  95. source = source.strip()
  96. kind, match = next(((k, m) for k, p in _BOOTSTRAPS if (m := p.fullmatch(source))), (None, None))
  97. if match is None:
  98. return None
  99. target = match["target"]
  100. if kind == "base64":
  101. import base64
  102. import binascii
  103. try:
  104. return _bootstrap_entry(base64.b64decode(target, validate=True).decode("utf-8"), argv)
  105. except (binascii.Error, UnicodeDecodeError):
  106. return None
  107. if kind == "exec":
  108. # The launcher source is a string literal here, and readers normalize its escapes (``\\n`` ->
  109. # ``/n``), so it is matched in place by the launcher row's anchors, never decoded.
  110. if assigned := _ASSIGNED_ARGV.search(source):
  111. argv = [item.strip().strip("'\"") for item in assigned.group(1).split(",")][1:]
  112. if wrapped := _EXEC_BASE64.search(target): # the .cmd launcher: its script, base64-encoded
  113. import base64
  114. import binascii
  115. try:
  116. return _bootstrap_entry(base64.b64decode(wrapped["b64"], validate=True).decode("utf-8"), argv)
  117. except (binascii.Error, UnicodeDecodeError):
  118. return None
  119. launcher = _EXEC_LAUNCHER.search(target)
  120. if launcher is None:
  121. return None
  122. kind, target = "entry", launcher["target"]
  123. if kind == "entry": # the launcher script's own ``--run-module <module>`` switch
  124. return ["-m", argv[1], *argv[2:]] if argv[:1] == ["--run-module"] and len(argv) > 1 else ["-m", target, *argv]
  125. if assigned := _ASSIGNED_ARGV.search(source):
  126. argv = [item.strip().strip("'\"") for item in assigned.group(1).split(",")][1:]
  127. return [target, *argv] if kind == "path" else ["-m", target, *argv]
  128. def inline_bootstrap_argv(tokens: list[str]) -> list[str] | None:
  129. """*tokens* as the equivalent ``python -m <module> <argv…>`` when this interpreter's ``-c`` source
  130. is a Hermes bootstrap running an entry point in-process; None for any other inline source.
  131. Command lines usually arrive space-joined (``/proc``, psutil, ``ps``), which splits the source
  132. across tokens; the shortest token run that ends in a recognised tail is the source, whatever
  133. joined it, and the tokens after it are the entry point's argv.
  134. """
  135. index = inline_source_flag_index(tokens)
  136. if index is None:
  137. return None
  138. for end in range(index + 1, len(tokens)):
  139. if tokens[end].rstrip().endswith(")"):
  140. entry = _bootstrap_entry(" ".join(tokens[index + 1 : end + 1]), tokens[end + 1 :])
  141. if entry is not None:
  142. return [tokens[0], *entry]
  143. return None