monitor.py 6.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186
  1. """Monitor-mode cron support — hash-suppressed change detection.
  2. A monitor job attaches a cheap source (``monitor_script`` / ``monitor_url``) to an LLM cron job.
  3. Each tick runs the source FIRST and hashes its EXACT output bytes (no timestamp/whitespace
  4. normalization — scripts must emit stable output) against the hash from the last agent-triggering
  5. tick: unchanged → agent run suppressed (silent ``no_change`` run); changed/first run → a "MONITOR
  6. CHANGE DETECTED" block (capped unified diff + new output) is injected into the prompt; source
  7. failure → an ERROR, never a change, and the stored hash is left untouched. State:
  8. ``job["monitor_state"]`` in jobs.json (hash + last_changed_at) and
  9. ``OUTPUT_DIR/<job_id>/monitor_last_output.txt`` (for the diff).
  10. """
  11. from __future__ import annotations
  12. import difflib
  13. import hashlib
  14. import logging
  15. from dataclasses import dataclass
  16. from typing import Optional
  17. logger = logging.getLogger(__name__)
  18. # Prompt-injection caps: unified diff, and new-output block (mirrors the 8k context_from truncation
  19. # in cron/scheduler.py). Then bounded-GET limits for monitor_url sources.
  20. MAX_DIFF_CHARS = 4000
  21. MAX_OUTPUT_CHARS = 8000
  22. URL_TIMEOUT_SECONDS = 30
  23. MAX_URL_BYTES = 262_144 # 256 KiB
  24. _SNAPSHOT_FILENAME = "monitor_last_output.txt"
  25. @dataclass
  26. class MonitorOutcome:
  27. """Result of one monitor-source evaluation."""
  28. ok: bool
  29. changed: bool = False
  30. first_run: bool = False
  31. context_block: Optional[str] = None
  32. error: Optional[str] = None
  33. def hash_monitor_output(output: str) -> str:
  34. """Hash the monitor output as exact UTF-8 bytes (no normalization)."""
  35. return hashlib.sha256(output.encode("utf-8", errors="replace")).hexdigest()
  36. def build_monitor_diff(old: str, new: str) -> str:
  37. """Unified diff of old vs new monitor output, capped at MAX_DIFF_CHARS."""
  38. diff = "\n".join(
  39. difflib.unified_diff(
  40. old.splitlines(), new.splitlines(), fromfile="previous", tofile="current", lineterm="",
  41. )
  42. )
  43. if len(diff) > MAX_DIFF_CHARS:
  44. diff = diff[:MAX_DIFF_CHARS] + "\n... [diff truncated]"
  45. return diff
  46. def _snapshot_path(job_id: str):
  47. from cron.jobs import _job_output_dir
  48. return _job_output_dir(job_id) / _SNAPSHOT_FILENAME
  49. def _read_last_output(job_id: str) -> str:
  50. try:
  51. path = _snapshot_path(job_id)
  52. if path.exists():
  53. return path.read_text(encoding="utf-8-sig")
  54. except Exception as exc:
  55. logger.warning("Monitor: failed to read last output for %r: %s", job_id, exc)
  56. return ""
  57. def _write_last_output(job_id: str, output: str) -> None:
  58. try:
  59. from cron.jobs import _ensure_cron_dir
  60. path = _snapshot_path(job_id)
  61. _ensure_cron_dir(path.parent)
  62. path.write_text(output, encoding="utf-8")
  63. except Exception as exc:
  64. logger.warning("Monitor: failed to persist last output for %r: %s", job_id, exc)
  65. def _fetch_monitor_url(url: str) -> tuple[bool, str]:
  66. """Bounded GET of a monitor URL. Returns (ok, body-or-error)."""
  67. import urllib.request
  68. if not str(url).lower().startswith(("http://", "https://")):
  69. return False, f"monitor_url must be http(s): {url!r}"
  70. try:
  71. req = urllib.request.Request(url, headers={"User-Agent": "hermes-cron-monitor"})
  72. with urllib.request.urlopen(req, timeout=URL_TIMEOUT_SECONDS) as resp: # nosec B310 — scheme checked above
  73. body = resp.read(MAX_URL_BYTES + 1)
  74. return True, body[:MAX_URL_BYTES].decode("utf-8", errors="replace")
  75. except Exception as exc:
  76. return False, f"monitor_url fetch failed: {exc}"
  77. def _field(job: dict, key: str) -> str:
  78. return (job.get(key) or "").strip()
  79. def _run_monitor_source(job: dict) -> tuple[bool, str]:
  80. """Run the job's monitor source (script or URL). Returns (ok, output)."""
  81. monitor_script = _field(job, "monitor_script")
  82. if monitor_script:
  83. # Same containment + interpreter rules as the existing `script` field.
  84. from cron.scheduler_script import _run_job_script
  85. return _run_job_script(monitor_script, workdir=_field(job, "workdir") or None,
  86. interpreter=job.get("interpreter"))
  87. monitor_url = _field(job, "monitor_url")
  88. if monitor_url:
  89. return _fetch_monitor_url(monitor_url)
  90. return False, "monitor job has neither monitor_script nor monitor_url"
  91. def job_has_monitor(job: dict) -> bool:
  92. return bool(_field(job, "monitor_script") or _field(job, "monitor_url"))
  93. def check_monitor(job: dict) -> MonitorOutcome:
  94. """Run the monitor source and decide whether the agent should run.
  95. On change (or first run) the new hash + snapshot are persisted BEFORE the agent runs — detection
  96. time is the state boundary, so a failed agent run doesn't re-alert on the same content forever.
  97. On failure nothing is persisted.
  98. """
  99. job_id = str(job.get("id") or "")
  100. ok, output = _run_monitor_source(job)
  101. if not ok:
  102. return MonitorOutcome(ok=False, error=output)
  103. new_hash = hash_monitor_output(output)
  104. raw_state = job.get("monitor_state")
  105. last_hash = raw_state.get("last_output_hash") if isinstance(raw_state, dict) else None
  106. if last_hash is not None and new_hash == last_hash:
  107. return MonitorOutcome(ok=True, changed=False)
  108. first_run = last_hash is None
  109. old_output = "" if first_run else _read_last_output(job_id)
  110. shown_output = output
  111. if len(shown_output) > MAX_OUTPUT_CHARS:
  112. shown_output = shown_output[:MAX_OUTPUT_CHARS] + "\n... [output truncated]"
  113. current = f"### Current output\n\n```\n{shown_output}\n```"
  114. if first_run:
  115. context_block = (
  116. "## Monitor Baseline (first run)\n\n"
  117. "This is the first observation of the monitored source — there is "
  118. "no previous output to diff against.\n\n" + current
  119. )
  120. else:
  121. diff = build_monitor_diff(old_output, output)
  122. context_block = (
  123. "## MONITOR CHANGE DETECTED\n\n"
  124. "The monitored source's output changed since the last run.\n\n"
  125. f"### Diff (previous → current)\n\n```diff\n{diff}\n```\n\n" + current
  126. )
  127. _persist_monitor_state(job_id, new_hash, output)
  128. return MonitorOutcome(ok=True, changed=True, first_run=first_run, context_block=context_block)
  129. def _persist_monitor_state(job_id: str, new_hash: str, output: str) -> None:
  130. from cron.jobs import _hermes_now, update_job
  131. _write_last_output(job_id, output)
  132. try:
  133. update_job(
  134. job_id,
  135. {
  136. "monitor_state": {
  137. "last_output_hash": new_hash,
  138. "last_changed_at": _hermes_now().isoformat(),
  139. }
  140. },
  141. )
  142. except Exception as exc:
  143. logger.warning("Monitor: failed to persist state for %r: %s", job_id, exc)