adapter.py 60 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031
  1. """Email platform adapter for the Hermes gateway: users talk to Hermes by sending email; IMAP (polled)
  2. receives, SMTP sends. Configured via EMAIL_* env vars or ``platforms.email`` in config.yaml (see website docs)."""
  3. import asyncio
  4. import email as email_lib
  5. from contextlib import contextmanager, suppress
  6. import imaplib
  7. import logging
  8. import os
  9. import re
  10. import smtplib
  11. import socket
  12. import ssl
  13. import uuid
  14. from email.header import decode_header
  15. from email.mime.multipart import MIMEMultipart
  16. from email.mime.text import MIMEText
  17. from email.mime.base import MIMEBase
  18. from email.parser import BytesHeaderParser
  19. from email.utils import formatdate, parseaddr
  20. from email import encoders
  21. from pathlib import Path
  22. from typing import Any, Callable, Dict, List, Optional, Tuple
  23. from agent.async_utils import safe_schedule_threadsafe
  24. from agent.i18n import t
  25. from gateway.platforms.base import (
  26. BasePlatformAdapter, SendResult,
  27. cache_document_from_bytes, cache_image_from_bytes,
  28. )
  29. from gateway.platforms.helpers import cancel_task
  30. from gateway.platforms.event import MessageEvent, MessageType
  31. from gateway.config import Platform, PlatformConfig
  32. from utils import is_truthy_value
  33. from gateway.platforms._shared import get_scoped_secret as _get_secret, coerce_port, decode_json_list_literal, send_error
  34. from gateway.platforms.send_targets import StandaloneSendPolicy, regex_stripped_parser
  35. logger = logging.getLogger(__name__)
  36. _SECURITY_ALIASES = {"tls": "tls", "ssl": "tls", "implicit": "tls", "starttls": "starttls", "plain": "plain", "none": "plain"}
  37. # Automated senders (address substrings / bulk-mail headers) are silently ignored.
  38. _NOREPLY_PATTERNS = ("noreply", "no-reply", "no_reply", "donotreply", "do-not-reply", "mailer-daemon", "postmaster",
  39. "bounce", "notifications@", "automated@", "auto-confirm", "auto-reply", "automailer")
  40. _AUTOMATED_HEADERS = {"Auto-Submitted": lambda v: v.lower() != "no",
  41. "Precedence": lambda v: v.lower() in {"bulk", "list", "junk"},
  42. "X-Auto-Response-Suppress": lambda v: bool(v), "List-Unsubscribe": lambda v: bool(v)}
  43. MAX_MESSAGE_LENGTH = 50_000 # Gmail-safe max length per email body
  44. SMTP_CONNECT_TIMEOUT = 30
  45. _TRUTHY = {"true", "1", "yes"}
  46. _IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".gif", ".webp"}
  47. # Charset labels seen in the wild that Python's codec registry doesn't know: "unknown-8bit"/"x-unknown" are
  48. # RFC 1428 placeholders (QQ Mail emits them); gb2312/gbk map to the gb18030 superset so GBK extensions decode.
  49. _CHARSET_ALIASES = {"unknown-8bit": "utf-8", "unknown": "utf-8", "x-unknown": "utf-8", "default": "utf-8",
  50. "ansi_x3.110-1983": "latin-1", "cp-850": "cp850",
  51. "gb2312": "gb18030", "gbk": "gb18030", "ks_c_5601-1987": "cp949"}
  52. # Ordered (pattern, replacement) substitutions for _strip_html.
  53. _HTML_SUBS = ((re.compile(r"<br\s*/?>", re.IGNORECASE), "\n"), (re.compile(r"<p[^>]*>", re.IGNORECASE), "\n"),
  54. (re.compile(r"</p>", re.IGNORECASE), "\n"), (re.compile(r"<[^>]+>"), ""), (re.compile(r"&nbsp;"), " "),
  55. (re.compile(r"&amp;"), "&"), (re.compile(r"&lt;"), "<"), (re.compile(r"&gt;"), ">"), (re.compile(r"\n{3,}"), "\n\n"))
  56. # ``display <bracketed>`` split for the _extract_email_address fallback (linear: neither part can match the other's delimiters).
  57. _SINGLE_BRACKET_FROM_RE = re.compile(r'([^"<>]*)<([^<>\s]+)>\s*')
  58. _COMMENT_RE = re.compile(r"\([^()]*\)")
  59. # Longest From: value we parse. parseaddr is pure Python and superlinear on hostile input (~1s at 100KB, GIL held);
  60. # a real mailbox plus display name stays far below this (RFC 5322 caps a line at 998 chars).
  61. _MAX_FROM_LEN = 2048
  62. # Only these headers are needed to reject a sender. Request one byte past the
  63. # limit so a truncated preflight fails closed without downloading the MIME body.
  64. _MAX_PREAUTH_HEADER_BYTES = 64 * 1024
  65. _PREAUTH_FETCH = (
  66. f"(BODY.PEEK[HEADER.FIELDS (FROM AUTHENTICATION-RESULTS {' '.join(h.upper() for h in _AUTOMATED_HEADERS)})]"
  67. f"<0.{_MAX_PREAUTH_HEADER_BYTES + 1}>)"
  68. )
  69. # Authentication-Results clause head (``dmarc=pass``), matched only at the start of a clause.
  70. _AUTH_METHOD_RE = re.compile(r"\s*(dmarc|dkim|spf)\s*=\s*([a-z]+)", re.IGNORECASE)
  71. _NO_AUTH_RESULTS_REASON = "no Authentication-Results header"
  72. _UNTRUSTED_AUTHSERV_REASON = "no Authentication-Results from trusted authserv-id"
  73. _MISSING_AUTHSERV_REASON = "authserv-id is not configured; refusing to trust Authentication-Results"
  74. _OPT_OUT_HINT = "set platforms.email.require_authenticated_sender: false (or EMAIL_TRUST_FROM_HEADER=true) to accept the risk."
  75. # Operator-fixable reasons a granted sender's mail fails authentication, and the fix each log line names.
  76. _DROP_HINTS = {
  77. _NO_AUTH_RESULTS_REASON: " If your mail server does not stamp Authentication-Results, " + _OPT_OUT_HINT,
  78. _UNTRUSTED_AUTHSERV_REASON: " Check that platforms.email.authserv_id (EMAIL_AUTHSERV_ID) names your mail server.",
  79. }
  80. # A missing pin is account config, not one sender's mail, so connect() names its fix once per account.
  81. _MISSING_AUTHSERV_HINT = (" Set EMAIL_AUTHSERV_ID (or platforms.email.authserv_id) to the receiving MTA's exact authserv-id, "
  82. "or " + _OPT_OUT_HINT)
  83. # One token of a clause: a property we read (``header.from=x``; the value may be or contain a quoted-string), or
  84. # any other whitespace-delimited token consumed whole, so text inside quotes or other values is never read as a prop.
  85. _QUOTED = r'"(?:[^"\\]|\\.)*"'
  86. _AUTH_PROP_RE = re.compile(rf'(header\.from|header\.d|smtp\.mailfrom|smtp\.from|envelope-from)\s*=\s*((?:{_QUOTED}|[^\s";])+)'
  87. rf'|(?:{_QUOTED}|[^\s"])+', re.IGNORECASE)
  88. def _esecret_int(name: str, default: int) -> int:
  89. """Scope-aware integer read."""
  90. return coerce_port(str(_get_secret(name, "")).strip() or default, default)
  91. def _esecret_bool(name: str, default: bool = False) -> bool:
  92. """Scope-aware boolean read."""
  93. return is_truthy_value(raw, default=default) if (raw := str(_get_secret(name, "")).strip()) else default
  94. def _normalize_security(value: Any, default: str = "tls") -> str:
  95. """Map to ``tls`` | ``starttls`` | ``plain``; unknown values warn and fall back to *default* (a typo never downgrades to plaintext)."""
  96. raw = str(value or "").strip().lower().replace("-", "").replace("_", "")
  97. if raw and raw not in _SECURITY_ALIASES:
  98. logger.warning("Unknown email security mode %r; using %r", value, default)
  99. return _SECURITY_ALIASES.get(raw, default)
  100. def _tls_context(verify: bool, host: str) -> ssl.SSLContext:
  101. """Verified context by default; unverified only when explicitly opted out."""
  102. if verify:
  103. return ssl.create_default_context()
  104. if host not in ("127.0.0.1", "::1", "localhost"):
  105. logger.warning("TLS verification disabled for non-loopback host %s", host)
  106. return ssl._create_unverified_context()
  107. def _close_imap(imap: imaplib.IMAP4) -> None:
  108. """Teardown that guarantees the socket closes: ``logout()`` only guards ``OSError``, so ``IMAP4.abort`` on a
  109. broken connection skipped ``shutdown()`` and leaked one fd per failed poll (fatal on macOS's 256 soft limit).
  110. ``IMAP4.logout()`` only guards against ``OSError`` internally: a broken connection makes
  111. ``_simple_command('LOGOUT')`` raise ``IMAP4.abort`` (which is *not* an ``OSError``), so ``logout()``
  112. propagates before its own ``shutdown()`` call and the TCP socket stays open. On macOS, where the default
  113. soft fd limit is 256 and pollers may run through a local proxy, these abandoned sockets accumulate one
  114. per failed poll until the gateway hits ``[Errno 24] Too many open files`` (#79889).
  115. """
  116. try:
  117. imap.logout()
  118. except Exception:
  119. with suppress(Exception):
  120. imap.shutdown()
  121. def _create_ipv4_connection(host: str, port: int, timeout: float, source_address: Any = None) -> socket.socket:
  122. """``socket.create_connection`` constrained to ``AF_INET`` (no process-global socket mutation — sends run in executor threads)."""
  123. last_error: OSError | None = None
  124. for family, socktype, proto, _canonname, sockaddr in socket.getaddrinfo(host, port, socket.AF_INET, socket.SOCK_STREAM):
  125. sock = socket.socket(family, socktype, proto)
  126. sock.settimeout(timeout)
  127. try:
  128. if source_address:
  129. sock.bind(source_address)
  130. sock.connect(sockaddr)
  131. return sock
  132. except OSError as exc:
  133. last_error = exc
  134. sock.close()
  135. raise last_error if last_error is not None else OSError(f"No IPv4 address found for {host}:{port}")
  136. class _IPv4SMTP(smtplib.SMTP):
  137. def _get_socket(self, host, port, timeout): # type: ignore[override]
  138. return _create_ipv4_connection(host, port, timeout, source_address=self.source_address)
  139. class _IPv4SMTP_SSL(smtplib.SMTP_SSL):
  140. def _get_socket(self, host, port, timeout): # type: ignore[override]
  141. return self.context.wrap_socket(_create_ipv4_connection(host, port, timeout, source_address=self.source_address), server_hostname=getattr(self, "_host", host))
  142. def _open_smtp(host: str, port: int, security: str, ctx: ssl.SSLContext, smtp_cls: type, smtp_ssl_cls: type, **kwargs: Any) -> smtplib.SMTP:
  143. """Open one SMTP connection with TLS established per *security*; *kwargs* go to the constructor."""
  144. if security == "tls":
  145. return smtp_ssl_cls(host, port, context=ctx, **kwargs)
  146. smtp = smtp_cls(host, port, **kwargs)
  147. if security == "starttls":
  148. try:
  149. smtp.starttls(context=ctx)
  150. except Exception:
  151. smtp.close()
  152. raise
  153. return smtp
  154. def _send_imap_id(imap: imaplib.IMAP4) -> None:
  155. """Send RFC 2971 IMAP ID: 163/NetEase require it after LOGIN (else every UID command
  156. returns ``BYE Unsafe Login``); other servers may reject it, so failures are swallowed.
  157. Sent only when the server advertises ``ID`` (RFC 2971 requires advertising it): a server
  158. without the extension can answer an untagged ``* BYE Unknown command.`` and close the
  159. connection, which imaplib cannot surface here — the failure appears one command later
  160. as a misleading SELECT error and the adapter retries forever (Purelymail, #39856).
  161. ``imap.capabilities`` is populated by imaplib at connect, so the check is free."""
  162. if "ID" not in imap.capabilities:
  163. logger.debug(
  164. "[Email] Server does not advertise IMAP ID capability; skipping ID"
  165. )
  166. return
  167. try:
  168. try:
  169. from hermes_cli.version_info import get_version_info
  170. version = get_version_info().base_version
  171. except Exception:
  172. version = "0"
  173. imap.xatom("ID", f'("name" "hermes-agent" "version" "{version}" '
  174. '"vendor" "NousResearch" "support-email" "noreply@nousresearch.com")')
  175. except Exception as e:
  176. logger.debug("[Email] IMAP ID command not accepted: %s", e)
  177. def _is_automated_sender(address: str, headers: dict) -> bool:
  178. """True if this email is from an automated/noreply source."""
  179. addr = address.lower()
  180. return any(pattern in addr for pattern in _NOREPLY_PATTERNS) or any(
  181. (value := headers.get(header, "")) and check(value) for header, check in _AUTOMATED_HEADERS.items())
  182. def _imap_payload(data: Any) -> Optional[bytes]:
  183. """The bytes of a one-message IMAP FETCH response, or ``None`` for an unexpected shape (see #80032)."""
  184. try:
  185. payload = data[0][1]
  186. except (IndexError, TypeError):
  187. return None
  188. return bytes(payload) if isinstance(payload, (bytes, bytearray)) else None
  189. def check_email_requirements() -> bool:
  190. """True when all email settings are present and non-blank (blank keys left by an abandoned setup must not enable the platform).
  191. Treats blank/whitespace-only values as missing so an abandoned setup that left empty ``EMAIL_*`` keys in
  192. ``.env`` does not enable the platform (#40715).
  193. """
  194. return all(_get_secret(name, "").strip() for name in ("EMAIL_ADDRESS", "EMAIL_PASSWORD", "EMAIL_IMAP_HOST", "EMAIL_SMTP_HOST"))
  195. def _safe_decode(payload: bytes, charset: Optional[str]) -> str:
  196. """Decode without ever raising: ``errors="replace"`` does not guard a missing codec (``LookupError``), so fall back alias → UTF-8 → latin-1.
  197. Unknown or malformed charset labels (``unknown-8bit``, misspelled names, attacker-controlled garbage)
  198. previously raised ``LookupError`` from ``bytes.decode`` — ``errors="replace"`` only guards decode
  199. errors, not a missing codec — which aborted the whole IMAP fetch and dropped every message in the batch
  200. (#35901, #55381, #55383). Fall back through a small alias table, then UTF-8, then latin-1 (which never
  201. fails).
  202. """
  203. label = (charset or "utf-8").strip().strip("\"'").lower() or "utf-8"
  204. for candidate in (_CHARSET_ALIASES.get(label, label), "utf-8"):
  205. try:
  206. return payload.decode(candidate, errors="replace")
  207. except (LookupError, ValueError):
  208. continue
  209. return payload.decode("latin-1", errors="replace")
  210. def _decode_header_value(raw: str) -> str:
  211. """Decode an RFC 2047 header into a plain string; never raises.
  212. Never raises: malformed encoded-words or unknown charsets degrade to replacement characters instead of
  213. crashing the fetch loop (#55381).
  214. """
  215. try:
  216. parts = decode_header(raw)
  217. except Exception: # malformed RFC 2047 structure
  218. return raw
  219. return " ".join(_safe_decode(part, charset) if isinstance(part, bytes) else part for part, charset in parts)
  220. def _first_body_part(msg: email_lib.message.Message, content_type: str) -> str:
  221. """Decoded text of the first non-attachment part of *content_type*, or ''."""
  222. for part in msg.walk():
  223. if "attachment" in str(part.get("Content-Disposition", "")) or part.get_content_type() != content_type:
  224. continue
  225. if payload := part.get_payload(decode=True):
  226. return _safe_decode(payload, part.get_content_charset())
  227. return ""
  228. def _extract_text_body(msg: email_lib.message.Message) -> str:
  229. """Extract the plain-text body from a potentially multipart email."""
  230. if msg.is_multipart():
  231. html = _first_body_part(msg, "text/html")
  232. return _first_body_part(msg, "text/plain") or (_strip_html(html) if html else "")
  233. text = _safe_decode(payload, msg.get_content_charset()) if (payload := msg.get_payload(decode=True)) else ""
  234. return _strip_html(text) if msg.get_content_type() == "text/html" else text
  235. def _strip_html(html: str) -> str:
  236. """Naive HTML tag stripper for fallback text extraction."""
  237. for pattern, repl in _HTML_SUBS:
  238. html = pattern.sub(repl, html)
  239. return html.strip()
  240. def _extract_email_address(raw: str) -> str:
  241. """Bare lowercased addr-spec from a From: value. Uses parseaddr, not a first-<...> regex (GHSA-rxqh-5572-8m77);
  242. RFC 5322 folding is unfolded first because parseaddr misreads a folded quoted display name. Unquoted
  243. ``Name <addr>`` values parseaddr rejects (``Doe, John <j@x>``) fall back to their single bracketed address,
  244. but only when the display part (``(comments)`` removed) cannot hold another mailbox or group: no quotes, ``;``,
  245. ``:`` or stray parens, and no ``@`` unless it is exactly the bracketed address (``a@x <a@x>``). Values over
  246. ``_MAX_FROM_LEN`` (or with more than 64 ``(``) and results without ``@`` return ``""`` so the caller drops the message."""
  247. value = re.sub(r"\r?\n[ \t]+", " ", str(raw or ""))
  248. if len(value) > _MAX_FROM_LEN or value.count("(") > 64:
  249. return "" # hostile size/nesting: take the empty-sender drop (parseaddr recurses per nested comment)
  250. _, addr = parseaddr(value)
  251. if not addr and (m := _SINGLE_BRACKET_FROM_RE.fullmatch(value)):
  252. display, bracketed = _strip_comments(m.group(1)).strip(), m.group(2)
  253. if ("@" in bracketed[1:-1] and not any(c in display for c in ";:()")
  254. and ("@" not in display or display.lower() == bracketed.lower())):
  255. addr = bracketed
  256. addr = addr.strip().lower()
  257. return addr if "@" in addr else "" # a bare word (``John``) is not a sender identity
  258. def _strip_comments(text: str) -> str:
  259. """Remove (possibly nested) ``(comments)``, innermost first, until nothing changes."""
  260. while (stripped := _COMMENT_RE.sub(" ", text)) != text:
  261. text = stripped
  262. return text
  263. def _ar_clauses(text: str) -> Optional[list[str]]:
  264. """Split an Authentication-Results value on ``;`` outside quoted-strings and (nested) comments; comments are
  265. dropped, quoted-strings kept (``header.from="x"`` stays readable). ``None`` when a quote or comment is unbalanced."""
  266. clauses, cur, depth, quoted, i = [], [], 0, False, 0
  267. while i < len(text):
  268. c = text[i]
  269. if c == "\\" and (quoted or depth):
  270. if quoted:
  271. cur.append(text[i:i + 2])
  272. i += 2
  273. continue
  274. if quoted:
  275. quoted = c != '"'
  276. cur.append(c)
  277. elif depth:
  278. depth += {"(": 1, ")": -1}.get(c, 0)
  279. if not depth:
  280. cur.append(" ")
  281. elif c == "(":
  282. depth = 1
  283. elif c == '"':
  284. quoted = True
  285. cur.append(c)
  286. elif c == ")":
  287. return None # stray close paren: unbalanced
  288. elif c == ";":
  289. clauses.append("".join(cur))
  290. cur = []
  291. else:
  292. cur.append(c)
  293. i += 1
  294. return None if quoted or depth else clauses + ["".join(cur)]
  295. def _auth_props(text: str) -> list[tuple[str, str]]:
  296. """``(property, value)`` pairs (``header.from=x``) of one comment-free Authentication-Results clause, property
  297. lowercased, surrounding quotes stripped. Quoted-string contents are never scanned for properties."""
  298. return [(p.lower(), v.strip('"')) for p, v in _AUTH_PROP_RE.findall(text) if p]
  299. def _domain_of(address: str) -> str:
  300. """Lowercased domain part of an email address, or ''."""
  301. return address.rpartition("@")[2].strip().lower()
  302. def _domains_aligned(a: str, b: str) -> bool:
  303. """Relaxed DMARC alignment: equal, or one is a dot-suffix of the other."""
  304. a = (a or "").strip().lower().rstrip(".")
  305. b = (b or "").strip().lower().rstrip(".")
  306. return bool(a and b) and (a == b or a.endswith("." + b) or b.endswith("." + a))
  307. def _verify_sender_authentication(msg: email_lib.message.Message, from_addr: str, *, authserv_id: str) -> tuple[bool, str]:
  308. """Verify the ``From:`` domain is authenticated; returns ``(authenticated, reason)``.
  309. ``From:`` is attacker-controlled (GHSA-rxqh-5572-8m77); the only trustworthy signal is the
  310. ``Authentication-Results`` header stamped by the *receiving* server. It prepends, so only the
  311. FIRST instance is authoritative, and it must match the required, already-normalised *authserv_id* exactly. A matching id is a
  312. pin, not proof of provenance: the receiving MTA must strip inbound results claiming its id (RFC 8601).
  313. True on DMARC pass, aligned SPF pass, or aligned DKIM (``header.d``) pass. No header or no pin → fail-closed
  314. (opt out via ``EmailAdapter._require_authenticated_sender``)."""
  315. from_domain = _domain_of(from_addr)
  316. if not from_domain:
  317. return False, "missing From domain"
  318. if not (headers := msg.get_all("Authentication-Results")):
  319. return False, _NO_AUTH_RESULTS_REASON
  320. trusted = " ".join(str(headers[0]).split())
  321. # _ar_clauses removes RFC 8601 CFWS comments and ignores semicolons inside them, so supported
  322. # receiver variants remain valid without allowing a lower field or a related domain to satisfy the pin.
  323. if (clauses := _ar_clauses(trusted)) is None:
  324. return False, "unbalanced quote or comment in Authentication-Results"
  325. if not authserv_id: # without a pin the topmost header may be one the sender wrote
  326. return False, _MISSING_AUTHSERV_REASON
  327. if clauses[0].strip().lower() != authserv_id:
  328. return False, _UNTRUSTED_AUTHSERV_REASON
  329. # Each verdict comes from the head of its own clause (split outside quotes/comments) and its domains only from that
  330. # clause: a quoted local part or comment can otherwise smuggle ``spf=pass``/``header.d=`` (GHSA-rxqh-5572-8m77).
  331. results: dict[str, list[tuple[str, list[tuple[str, str]]]]] = {"dmarc": [], "spf": [], "dkim": []}
  332. for clause in clauses:
  333. if m := _AUTH_METHOD_RE.match(clause):
  334. results[m.group(1).lower()].append((m.group(2).lower(), _auth_props(clause)))
  335. def aligned(props: list[tuple[str, str]], names: tuple[str, ...], *, required: bool = True) -> bool:
  336. domains = [_domain_of(v) for p, v in props if p in names]
  337. return (bool(domains) or not required) and all(_domains_aligned(d, from_domain) for d in domains)
  338. if len(results["dmarc"]) > 1:
  339. return False, "ambiguous dmarc result"
  340. # every header.from in the dmarc clause must be the From domain we parsed (absent header.from: trust the verdict)
  341. if any(r == "pass" and aligned(props, ("header.from",), required=False) for r, props in results["dmarc"]):
  342. return True, "dmarc=pass"
  343. # one SMTP transaction has one MAIL FROM verdict: a second spf clause means the SPF signal is not trusted
  344. if len(results["spf"]) == 1 and (spf := results["spf"][0])[0] == "pass" and aligned(
  345. spf[1], ("smtp.mailfrom", "smtp.from", "envelope-from")):
  346. return True, "spf=pass aligned"
  347. # several dkim clauses are normal (one per signature): any single pass whose own header.d aligns is enough
  348. if any(r == "pass" and aligned(props, ("header.d",) if any(p == "header.d" for p, _ in props) else ("header.from",))
  349. for r, props in results["dkim"]):
  350. return True, "dkim=pass aligned"
  351. return False, f"authentication failed ({trusted[:120]})"
  352. def _extract_attachments(msg: email_lib.message.Message, skip_attachments: bool = False) -> list[dict[str, Any]]:
  353. """Extract attachment metadata and cache files locally (nothing when *skip_attachments*)."""
  354. attachments = []
  355. if not msg.is_multipart():
  356. return attachments
  357. for part in msg.walk():
  358. disposition, content_type = str(part.get("Content-Disposition", "")), part.get_content_type()
  359. if skip_attachments or ("attachment" not in disposition and (
  360. "inline" not in disposition or content_type in {"text/plain", "text/html"})):
  361. continue # not an attachment, or an inline text/html body part
  362. filename = _decode_header_value(fn) if (fn := part.get_filename()) else f"attachment.{part.get_content_subtype() or 'bin'}"
  363. if not (payload := part.get_payload(decode=True)):
  364. continue
  365. if (ext := Path(filename).suffix.lower()) in _IMAGE_EXTS:
  366. try:
  367. cached_path, kind = cache_image_from_bytes(payload, ext), "image"
  368. except ValueError:
  369. logger.debug("Skipping non-image attachment %s (invalid magic bytes)", filename)
  370. continue
  371. else:
  372. cached_path, kind = cache_document_from_bytes(payload, filename), "document"
  373. attachments.append({"path": cached_path, "filename": filename, "type": kind, "media_type": content_type})
  374. return attachments
  375. def _attach_file(msg: MIMEMultipart, path: Path, filename: str) -> None:
  376. """Attach *path* to *msg* as base64 application/octet-stream."""
  377. with open(path, "rb") as f:
  378. part = MIMEBase("application", "octet-stream")
  379. part.set_payload(f.read())
  380. encoders.encode_base64(part)
  381. part.add_header("Content-Disposition", f"attachment; filename={filename}")
  382. msg.attach(part)
  383. class EmailAdapter(BasePlatformAdapter):
  384. """Email gateway adapter using IMAP (receive) and SMTP (send)."""
  385. serial_receive_loop = True # the IMAP poll loop awaits each message before dispatching the next
  386. # One email carries the whole body, so cron delivery hands over the full payload untruncated.
  387. splits_long_messages = True
  388. # Per-account seen-UID snapshot surviving adapter recreation: the reconnect watcher builds a FRESH
  389. # adapter per retry; without this connect(is_reconnect=True) would re-mark the mailbox seen and skip
  390. # mail that arrived during the outage. Keyed by address (multiplex runs several accounts); same-process only.
  391. _seen_uids_snapshot: dict[str, set] = {}
  392. # Accounts already warned about a missing authserv_id pin. Per address, not per first connect: an account whose
  393. # first connect fails is brought up by the reconnect watcher (is_reconnect=True) and must still warn once.
  394. _missing_pin_warned: set = set()
  395. def __init__(self, config: PlatformConfig):
  396. super().__init__(config, Platform.EMAIL)
  397. # Env first, then PlatformConfig.extra (config.yaml-only setups). Host/address are stripped: a stray
  398. # newline made IMAP4_SSL raise ``[Errno 8] nodename nor servname`` instead of "host not set".
  399. extra = config.extra or {}
  400. setting = lambda env, key: _get_secret(env, "") or extra.get(key, "")
  401. tls_verify = lambda env, key: _esecret_bool(env, is_truthy_value(extra.get(key), default=True))
  402. self._address = setting("EMAIL_ADDRESS", "address").strip()
  403. self._password = _get_secret("EMAIL_PASSWORD", "")
  404. self._imap_host = setting("EMAIL_IMAP_HOST", "imap_host").strip()
  405. self._imap_port = _esecret_int("EMAIL_IMAP_PORT", 993)
  406. self._imap_security = _normalize_security(setting("EMAIL_IMAP_SECURITY", "imap_security"))
  407. self._imap_tls_verify = tls_verify("EMAIL_IMAP_TLS_VERIFY", "imap_tls_verify")
  408. self._smtp_host = setting("EMAIL_SMTP_HOST", "smtp_host").strip()
  409. self._smtp_port = _esecret_int("EMAIL_SMTP_PORT", 587)
  410. self._smtp_security = _normalize_security(setting("EMAIL_SMTP_SECURITY", "smtp_security"), default="tls" if self._smtp_port == 465 else "starttls")
  411. self._smtp_tls_verify = tls_verify("EMAIL_SMTP_TLS_VERIFY", "smtp_tls_verify")
  412. self._poll_interval = _esecret_int("EMAIL_POLL_INTERVAL", 15)
  413. self._skip_attachments = extra.get("skip_attachments", False) # platforms.email.skip_attachments
  414. # Require an authenticated From: domain (SPF/DKIM/DMARC) before trusting it for authorization
  415. # (GHSA-rxqh-5572-8m77). Default ON; opt out via require_authenticated_sender: false / EMAIL_TRUST_FROM_HEADER=true.
  416. if "require_authenticated_sender" in extra:
  417. self._require_authenticated_sender = bool(extra["require_authenticated_sender"])
  418. else:
  419. self._require_authenticated_sender = not _esecret_bool("EMAIL_TRUST_FROM_HEADER", False)
  420. # Pin Authentication-Results to the operator's receiving MTA. With sender auth on, an absent pin fails
  421. # closed for EVERY sender rather than trusting a sender-supplied result header; connect() warns once per account.
  422. self._authserv_id = (extra.get("authserv_id", "") or _get_secret("EMAIL_AUTHSERV_ID", "")).strip().lower()
  423. self._seen_uids: set = set()
  424. self._seen_uids_max: int = 2000 # cap to prevent unbounded memory growth
  425. self._poll_task: Optional[asyncio.Task] = None
  426. self._last_fetch_failed, self._last_fetch_error = False, "" # "checked, nothing new" vs "the check itself failed"
  427. # chat_id (sender email) -> last subject + message-id for threading
  428. # Track the last IMAP fetch attempt so the poll loop can distinguish "checked, nothing new" from
  429. # "the check itself failed" (#80016).
  430. self._thread_context: dict[str, dict[str, str]] = {}
  431. logger.info("[Email] Adapter initialized for %s", self._address)
  432. def _trim_seen_uids(self) -> None:
  433. """Keep only the highest half of UIDs once over the cap (UIDs are monotonic; UNSEEN prevents re-delivery)."""
  434. if len(self._seen_uids) <= self._seen_uids_max:
  435. return
  436. try:
  437. sorted_uids = sorted(self._seen_uids, key=lambda u: int(u)) # UIDs are bytes like b'1234'
  438. self._seen_uids = set(sorted_uids[-(self._seen_uids_max // 2):])
  439. logger.debug("[Email] Trimmed seen UIDs to %d entries", len(self._seen_uids))
  440. except (ValueError, TypeError):
  441. self._seen_uids = set(list(self._seen_uids)[-self._seen_uids_max // 2:])
  442. def _connect_imap(self) -> imaplib.IMAP4:
  443. """Create an IMAP connection using implicit TLS, STARTTLS, or plaintext."""
  444. if self._imap_security == "tls":
  445. return imaplib.IMAP4_SSL(self._imap_host, self._imap_port, timeout=30, ssl_context=_tls_context(self._imap_tls_verify, self._imap_host))
  446. imap = imaplib.IMAP4(self._imap_host, self._imap_port, timeout=30)
  447. if self._imap_security == "starttls":
  448. try:
  449. imap.starttls(ssl_context=_tls_context(self._imap_tls_verify, self._imap_host))
  450. except Exception:
  451. _close_imap(imap)
  452. raise
  453. return imap
  454. @contextmanager
  455. def _inbox(self):
  456. """Logged-in IMAP handle on INBOX; always ``_close_imap``-ed on exit (a login/select failure used to leak one fd per reconnect)."""
  457. # Test IMAP connection. The handle is closed in ``finally`` — before this, a failure in
  458. # login/select/search left the TCP socket open with no owner, leaking one fd per connect attempt.
  459. # Under the gateway's reconnect watcher (fresh adapter instance per retry) against an
  460. # unreachable/proxied host this grew monotonically until fd exhaustion on macOS's 256 soft limit
  461. # (#79889).
  462. imap = self._connect_imap()
  463. try:
  464. imap.login(self._address, self._password)
  465. _send_imap_id(imap)
  466. imap.select("INBOX")
  467. yield imap
  468. finally:
  469. _close_imap(imap)
  470. def _connect_smtp(self) -> smtplib.SMTP:
  471. """SMTP connection with TLS established (callers go straight to ``login()``). An unreachable IPv6 address can
  472. hang until the socket timeout, so connection-level failures retry through an IPv4-only socket path (no global
  473. resolver mutation); TLS verification errors are not retried."""
  474. host, port, security, ctx = self._smtp_host, self._smtp_port, self._smtp_security, _tls_context(self._smtp_tls_verify, self._smtp_host)
  475. try:
  476. return _open_smtp(host, port, security, ctx, smtplib.SMTP, smtplib.SMTP_SSL, timeout=SMTP_CONNECT_TIMEOUT)
  477. except (TimeoutError, ConnectionError, OSError) as exc:
  478. if isinstance(exc, ssl.SSLError):
  479. raise
  480. return _open_smtp(host, port, security, ctx, _IPv4SMTP, _IPv4SMTP_SSL, timeout=SMTP_CONNECT_TIMEOUT)
  481. def _fail(self, log_fmt: str, err: object, code: str, detail: str, *, retryable: bool) -> bool:
  482. """Log *err*, record a fatal error for the gateway's reconnect machinery, return False."""
  483. logger.error(log_fmt, err)
  484. self._set_fatal_error(code, detail, retryable=retryable)
  485. return False
  486. def _probe_imap(self, is_reconnect: bool) -> bool:
  487. """Connection test + seen-UID baseline. Sets a fatal error and returns False on failure."""
  488. try:
  489. with self._inbox() as imap:
  490. snapshot = self._seen_uids_snapshot.get(self._address)
  491. if is_reconnect and snapshot is not None:
  492. # Same-process reconnect: restore the previous adapter's baseline so mail that
  493. # arrived during the outage stays eligible for the next poll.
  494. self._seen_uids = set(snapshot)
  495. passed = "[Email] IMAP reconnect test passed. Restored %d seen UIDs; messages received during the outage will be processed."
  496. else: # first connect (or no snapshot): mark all existing messages seen
  497. status, data = imap.uid("search", None, "ALL")
  498. self._seen_uids.update(data[0].split() if status == "OK" and data and data[0] else ())
  499. passed = "[Email] IMAP connection test passed. %d existing messages skipped."
  500. self._trim_seen_uids()
  501. logger.info(passed, len(self._seen_uids))
  502. self._seen_uids_snapshot[self._address] = set(self._seen_uids)
  503. return True
  504. except Exception as e:
  505. # Always set an explicit fatal code, else the gateway treats every failure as transient with zero
  506. # owner signal. retryable=True because imaplib raises the same generic IMAP4.error for bad credentials
  507. # AND transient NOs (Gmail "too many simultaneous connections"); loops surface via NEEDS_ATTENTION.
  508. return self._fail("[Email] IMAP connection failed: %s", e, "email_imap_connect_error",
  509. f"IMAP connection to {self._imap_host}:{self._imap_port} failed: {e}", retryable=True)
  510. def _probe_smtp(self) -> bool:
  511. """SMTP connect + login test. Sets a fatal error and returns False on failure."""
  512. try:
  513. smtp = self._connect_smtp()
  514. try:
  515. smtp.login(self._address, self._password)
  516. finally:
  517. smtp.quit()
  518. logger.info("[Email] SMTP connection test passed.")
  519. return True
  520. except smtplib.SMTPAuthenticationError as e:
  521. # Typed auth failure (535 & friends) can never self-heal, so drop out of the reconnect queue — unambiguous, unlike IMAP4.error.
  522. return self._fail("[Email] SMTP authentication failed: %s", e, "email_auth_error",
  523. f"SMTP authentication failed for {self._address}: {e}. Check EMAIL_PASSWORD (for Gmail/Outlook "
  524. "this must be an app password, not the account password).", retryable=False)
  525. except Exception as e:
  526. return self._fail("[Email] SMTP connection failed: %s", e, "email_smtp_connect_error",
  527. f"SMTP connection to {self._smtp_host} failed: {e}", retryable=True)
  528. async def connect(self, *, is_reconnect: bool = False) -> bool:
  529. """Connect to the IMAP server and start polling for new messages."""
  530. # Validate up front so a missing host is an actionable config error, not IMAP4_SSL("") raising ``[Errno 8]``.
  531. required = (("EMAIL_ADDRESS", self._address), ("EMAIL_PASSWORD", self._password), ("EMAIL_IMAP_HOST", self._imap_host), ("EMAIL_SMTP_HOST", self._smtp_host))
  532. if missing := [name for name, value in required if not value]:
  533. message = f"Not configured — missing {', '.join(missing)}. Set it via `hermes gateway setup` (env) or platforms.email in config.yaml."
  534. # Non-retryable: a blank-but-present env var used to drive an indefinite retry loop that leaked until OOM.
  535. return self._fail("[Email] %s", message, "email_missing_configuration", message, retryable=False)
  536. if not self._probe_imap(is_reconnect) or not self._probe_smtp():
  537. return False
  538. if self._require_authenticated_sender and not self._authserv_id and self._address not in EmailAdapter._missing_pin_warned:
  539. EmailAdapter._missing_pin_warned.add(self._address)
  540. logger.warning("[Email] %s: %s.%s", self._address, _MISSING_AUTHSERV_REASON, _MISSING_AUTHSERV_HINT)
  541. self._running = True
  542. self._poll_task = asyncio.create_task(self._poll_loop())
  543. print(f"[Email] Connected as {self._address}")
  544. self._wire_plugin_handlers(None) # plugin-registered native handlers
  545. return True
  546. async def disconnect(self) -> None:
  547. """Stop polling and disconnect."""
  548. self._running = False
  549. await cancel_task(self._poll_task)
  550. self._poll_task = None
  551. logger.info("[Email] Disconnected.")
  552. async def _poll_loop(self) -> None:
  553. """Poll IMAP for new messages at regular intervals."""
  554. while self._running:
  555. try:
  556. await self._check_inbox()
  557. except asyncio.CancelledError:
  558. break
  559. except Exception as e:
  560. logger.error("[Email] Poll error: %s", e)
  561. await asyncio.sleep(self._poll_interval)
  562. async def _check_inbox(self) -> None:
  563. """Check INBOX for unseen messages and dispatch them."""
  564. loop = asyncio.get_running_loop()
  565. async def authorize_on_loop(msg_data: dict[str, Any]) -> bool:
  566. return self._sender_accepted(msg_data["sender_addr"], msg_data)
  567. def gate(candidate: dict[str, Any]) -> bool:
  568. # Authorization reads profile-scoped policy and pairing state on the adapter's event loop;
  569. # the blocking IMAP worker waits for that verdict before asking the server for RFC822.
  570. # A closed loop (stop race) yields None: fail closed without leaking the coroutine.
  571. fut = safe_schedule_threadsafe(authorize_on_loop(candidate), loop, logger=logger,
  572. log_message="[Email] Could not schedule sender authorization")
  573. return fut is not None and fut.result()
  574. messages = await asyncio.to_thread(self._fetch_new_messages, gate)
  575. # Dispatch partial results BEFORE escalating a failure — a mid-batch exception returns what was fetched (already marked seen).
  576. for msg_data in messages:
  577. await self._dispatch_message(msg_data)
  578. if self._last_fetch_failed:
  579. # The IMAP check itself failed (not an empty inbox): route through the fatal-error hook so the gateway's
  580. # reconnect/backoff re-establishes the mailbox. The handler runs detached (gateway/run.py), so awaiting it is safe.
  581. # The handler runs in a detached task (gateway/run.py), so awaiting it from our own poll task is
  582. # safe even though teardown cancels this task. See #80016.
  583. self._last_fetch_failed = False
  584. self._set_fatal_error("email_imap_fetch_failed", self._last_fetch_error or "IMAP fetch failed", retryable=True)
  585. await self._notify_fatal_error()
  586. def _mark_uid_consumed(self, imap: imaplib.IMAP4, uid: Any) -> None:
  587. """Remember a rejected UID and mark it seen without fetching its MIME body."""
  588. self._seen_uids.add(uid)
  589. self._trim_seen_uids()
  590. status, _ = imap.uid("store", uid, "+FLAGS.SILENT", r"(\Seen)")
  591. if status != "OK":
  592. logger.warning("[Email] Could not mark rejected UID %s seen", uid)
  593. def _fetch_new_messages(self, preauthorize: Callable[[dict[str, Any]], bool]) -> list[dict[str, Any]]:
  594. """Fetch unseen messages; bounded headers pass *preauthorize* before RFC822 is requested."""
  595. results = []
  596. try:
  597. with self._inbox() as imap:
  598. status, data = imap.uid("search", None, "UNSEEN")
  599. for uid in (data[0].split() if status == "OK" and data and data[0] else []):
  600. if uid in self._seen_uids:
  601. continue
  602. header_status, header_data = imap.uid("fetch", uid, _PREAUTH_FETCH)
  603. if header_status != "OK":
  604. continue
  605. raw_headers = _imap_payload(header_data)
  606. if raw_headers is None or len(raw_headers) > _MAX_PREAUTH_HEADER_BYTES:
  607. logger.warning("[Email] Unusable pre-authorization headers for UID %s, skipping", uid)
  608. self._mark_uid_consumed(imap, uid)
  609. continue
  610. try:
  611. candidate = self._parse_fetched_headers(uid, raw_headers)
  612. accepted = candidate is not None and preauthorize(candidate)
  613. except Exception as auth_exc:
  614. logger.error("[Email] Failed to authorize message UID %s, skipping: %s", uid, auth_exc)
  615. accepted = False
  616. if not accepted:
  617. self._mark_uid_consumed(imap, uid)
  618. continue
  619. status, msg_data = imap.uid("fetch", uid, "(RFC822)")
  620. if status != "OK":
  621. continue # transient per-UID refusal: leave unseen so the next poll retries
  622. # Mark seen once a response arrived (even malformed) so garbage is skipped once, not retried forever —
  623. # but NOT before the fetch: a connection failure must leave the rest of the batch eligible for the next poll.
  624. self._seen_uids.add(uid)
  625. self._trim_seen_uids()
  626. if (raw_email := _imap_payload(msg_data)) is None:
  627. logger.warning("[Email] Unexpected IMAP response structure for UID %s, skipping", uid)
  628. continue
  629. # One poison message (unparseable headers, pathological attachment, DNS hiccup) must not abort the batch or force a reconnect.
  630. try:
  631. # See #80032.
  632. parsed = self._parse_fetched_message(uid, raw_email)
  633. except Exception as parse_exc:
  634. logger.error("[Email] Failed to process message UID %s, skipping: %s", uid, parse_exc)
  635. continue
  636. if parsed is not None:
  637. results.append(parsed)
  638. except Exception as e:
  639. # _close_imap guarantees the socket dies even when logout() raises IMAP4.abort on a broken
  640. # connection (#79889).
  641. logger.error("[Email] IMAP fetch error: %s", e)
  642. self._last_fetch_failed, self._last_fetch_error = True, str(e)
  643. # Keep the reconnect snapshot current so a mid-outage adapter recreation does not re-dispatch messages already processed.
  644. self._seen_uids_snapshot[self._address] = set(self._seen_uids)
  645. return results
  646. def _message_metadata(self, uid: bytes, msg: email_lib.message.Message) -> Optional[dict[str, Any]]:
  647. """Parse sender-facing headers and authentication without touching the MIME body."""
  648. if not (sender_addr := _extract_email_address(msg.get("From", ""))): # never dispatch an empty identity
  649. logger.debug("[Email] Dropping message with no parseable From address: %r", msg.get("From", ""))
  650. return None
  651. sender_name = _decode_header_value(msg.get("From", ""))
  652. if "<" in sender_name:
  653. sender_name = sender_name.split("<")[0].strip().strip('"')
  654. subject = _decode_header_value(msg.get("Subject", "(no subject)"))
  655. if _is_automated_sender(sender_addr, dict(msg.items())):
  656. logger.debug("[Email] Skipping automated sender: %s", sender_addr)
  657. return None
  658. sender_authenticated, auth_reason = _verify_sender_authentication(msg, sender_addr, authserv_id=self._authserv_id)
  659. return {"uid": uid, "sender_addr": sender_addr, "sender_name": sender_name, "subject": subject,
  660. "message_id": msg.get("Message-ID", ""), "in_reply_to": msg.get("In-Reply-To", ""),
  661. "date": msg.get("Date", ""), "sender_authenticated": sender_authenticated, "auth_reason": auth_reason}
  662. def _parse_fetched_headers(self, uid: bytes, raw_headers: bytes | bytearray) -> Optional[dict[str, Any]]:
  663. """Parse the bounded IMAP header preflight without constructing a MIME tree."""
  664. return self._message_metadata(uid, BytesHeaderParser().parsebytes(bytes(raw_headers)))
  665. def _parse_fetched_message(self, uid: bytes, raw_email: bytes | bytearray) -> Optional[dict[str, Any]]:
  666. """Parse an authorized RFC822 payload into a dispatchable dict."""
  667. msg = email_lib.message_from_bytes(raw_email)
  668. if (metadata := self._message_metadata(uid, msg)) is None:
  669. return None
  670. metadata.update(body=_extract_text_body(msg),
  671. attachments=_extract_attachments(msg, skip_attachments=self._skip_attachments))
  672. return metadata
  673. @staticmethod
  674. def _allow_all_senders() -> bool:
  675. """True when the operator opted into any sender (EMAIL_ or GATEWAY_ALLOW_ALL_USERS).
  676. Both names go through the scoped reader: under multiplex ``os.environ`` is the DEFAULT
  677. profile's opt-in, and borrowing it opened every secondary mailbox to any sender."""
  678. return any(_get_secret(name, "").strip().lower() in _TRUTHY
  679. for name in ("EMAIL_ALLOW_ALL_USERS", "GATEWAY_ALLOW_ALL_USERS"))
  680. def _answers_unknown_senders(self) -> bool:
  681. """True when ``platforms.email.unauthorized_dm_behavior`` opts into ``pair`` or ``decline``."""
  682. behavior = (self.config.extra or {}).get("unauthorized_dm_behavior")
  683. return isinstance(behavior, str) and behavior.strip().lower() in {"pair", "decline"}
  684. def _sender_accepted(self, sender_addr: str, msg_data: dict[str, Any]) -> bool:
  685. """Pre-dispatch sender gate: self, automated, authorization, From: authentication."""
  686. if sender_addr == self._address.lower():
  687. return False
  688. if _is_automated_sender(sender_addr, {}):
  689. logger.debug("[Email] Dropping automated sender at dispatch: %s", sender_addr)
  690. return False
  691. allowed_raw = _get_secret("EMAIL_ALLOWED_USERS", "").strip()
  692. # Parsed like the gateway's allowlists (JSON list literals included), or '["alice"]' would dodge the guard below.
  693. listed = set()
  694. for raw in (allowed_raw, _get_secret("GATEWAY_ALLOWED_USERS", "")):
  695. raw = decode_json_list_literal(raw)
  696. listed.update(str(a).strip().lower() for a in (raw if isinstance(raw, list) else str(raw).split(","))
  697. if str(a).strip())
  698. is_listed = sender_addr.lower() in listed
  699. if is_listed:
  700. granted = True
  701. elif sender_addr.split("@", 1)[0].lower() in listed:
  702. # The gateway's check also matches an address by its bare local part (#119446), so an entry like "alice"
  703. # would admit, or pair, alice@<any domain>; the domain is the sender's to choose.
  704. logger.debug("[Email] Dropping sender whose local part alone matches an allowlist entry: %s", sender_addr)
  705. return False
  706. else:
  707. # Approved pairings grant access too, and only the gateway's own check sees them. Its verdict also decides
  708. # open access: GATEWAY_ALLOW_ALL_USERS beside a GATEWAY_ALLOWED_USERS list grants a stranger nothing there.
  709. verdict = self._is_sender_authorized(sender_addr, "dm", sender_addr)
  710. granted = verdict if verdict is not None else (not allowed_raw and self._allow_all_senders())
  711. # Drop senders the gateway would neither authorize nor answer (pair/decline) before a MessageEvent (and thread
  712. # context) exists — otherwise a dispatch/authorization race can send a reply even though the handler returned None.
  713. if not granted and not self._answers_unknown_senders():
  714. logger.debug("[Email] Dropping unauthorized sender at dispatch (unknown senders are ignored): %s", sender_addr)
  715. return False
  716. # Reject spoofed senders (GHSA-rxqh-5572-8m77): every grant keys on the attacker-controlled From:, and a pairing
  717. # code or decline is mailed back to it; fail-closed. Open access is no exception: the session and every reply
  718. # key on From:, so a forged one lands in that address's conversation and makes the agent mail it.
  719. # Warn only where the operator can act: a known misconfiguration, or a listed contact's own mail failing.
  720. # Forged stranger mail (open access grants everyone) stays at debug.
  721. if self._require_authenticated_sender and not msg_data.get("sender_authenticated", False):
  722. auth_reason = msg_data.get("auth_reason", "no verdict")
  723. hint = _DROP_HINTS.get(auth_reason, "")
  724. if is_listed or (granted and hint):
  725. logger.warning("[Email] Dropping sender with unauthenticated From: %s (%s).%s", sender_addr, auth_reason, hint)
  726. else:
  727. logger.debug("[Email] Dropping %s sender with unauthenticated From: %s (%s)",
  728. "authorized" if granted else "unknown", sender_addr, auth_reason)
  729. return False
  730. return True
  731. async def _dispatch_message(self, msg_data: dict[str, Any]) -> None:
  732. """Convert a fetched email into a MessageEvent and dispatch it."""
  733. sender_addr = msg_data["sender_addr"]
  734. if not self._sender_accepted(sender_addr, msg_data):
  735. return
  736. subject, body, attachments = msg_data["subject"], msg_data["body"].strip(), msg_data["attachments"]
  737. text = f"[Subject: {subject}]\n\n{body}" if subject and not subject.startswith("Re:") else body # subject unless reply
  738. # DOCUMENT wins over PHOTO for mixed attachments: run.py keys image handling off the per-path mime type regardless
  739. # of message_type, but document-context injection gates strictly on MessageType.DOCUMENT — so DOCUMENT surfaces both.
  740. kinds = {att["type"] for att in attachments}
  741. self._thread_context[sender_addr] = {"subject": subject, "message_id": msg_data["message_id"]}
  742. name = msg_data["sender_name"] or sender_addr
  743. event = MessageEvent(
  744. text=text or "(empty email)", message_id=msg_data["message_id"],
  745. message_type=MessageType.DOCUMENT if "document" in kinds else MessageType.PHOTO if "image" in kinds else MessageType.TEXT,
  746. source=self.build_source(chat_id=sender_addr, chat_name=name, chat_type="dm", user_id=sender_addr, user_name=name,
  747. message_id=msg_data["message_id"]),
  748. media_urls=[att["path"] for att in attachments], media_types=[att["media_type"] for att in attachments],
  749. reply_to_message_id=msg_data["in_reply_to"] or None)
  750. logger.info("[Email] New message from %s: %s", sender_addr, subject)
  751. await self.handle_message(event)
  752. async def _run_send(self, fn, args: tuple, log_fmt: str, *log_args) -> SendResult:
  753. """Run a blocking SMTP sender in the executor; wrap its Message-ID in a SendResult."""
  754. try:
  755. return SendResult(success=True, message_id=await asyncio.get_running_loop().run_in_executor(None, fn, *args))
  756. except Exception as e:
  757. logger.error(log_fmt, *log_args, e)
  758. return SendResult(success=False, error=str(e))
  759. async def send(self, chat_id: str, content: str, reply_to: Optional[str] = None, metadata: Optional[dict[str, Any]] = None) -> SendResult:
  760. """Send an email reply to the given address."""
  761. return await self._run_send(self._send_email, (chat_id, content, reply_to), "[Email] Send failed to %s: %s", chat_id)
  762. def _message_id_domain(self) -> str:
  763. """Domain for generated Message-IDs; ``localhost`` when EMAIL_ADDRESS lacks ``@``."""
  764. return (self._address.rsplit("@", 1)[-1] if "@" in self._address else "") or "localhost"
  765. def _new_reply(self, to_addr: str, body: str, reply_to_msg_id: Optional[str] = None, *,
  766. attach_empty_body: bool = False) -> tuple[MIMEMultipart, str, str]:
  767. """Build a threaded reply skeleton. Returns ``(msg, msg_id, subject)``."""
  768. msg, ctx = MIMEMultipart(), self._thread_context.get(to_addr, {})
  769. subject = ctx.get("subject", "Hermes Agent")
  770. if not subject.startswith("Re:"):
  771. subject = f"Re: {subject}"
  772. original_msg_id = reply_to_msg_id or ctx.get("message_id")
  773. threading = (("In-Reply-To", original_msg_id), ("References", original_msg_id)) if original_msg_id else ()
  774. msg_id = f"<hermes-{uuid.uuid4().hex[:12]}@{self._message_id_domain()}>"
  775. for key, value in (("From", self._address), ("To", to_addr), ("Subject", subject), *threading,
  776. ("Date", formatdate(localtime=True)), ("Message-ID", msg_id)):
  777. msg[key] = value
  778. if body or attach_empty_body:
  779. msg.attach(MIMEText(body, "plain", "utf-8"))
  780. return msg, msg_id, subject
  781. def _smtp_send(self, msg: MIMEMultipart) -> None:
  782. """Login, send, and always release the SMTP connection (quit, else close)."""
  783. smtp = self._connect_smtp()
  784. try:
  785. smtp.login(self._address, self._password)
  786. smtp.send_message(msg)
  787. finally:
  788. try:
  789. smtp.quit()
  790. except Exception:
  791. smtp.close()
  792. def _send_email(self, to_addr: str, body: str, reply_to_msg_id: Optional[str] = None) -> str:
  793. """Send an email via SMTP. Runs in executor thread."""
  794. msg, msg_id, subject = self._new_reply(to_addr, body, reply_to_msg_id, attach_empty_body=True)
  795. self._smtp_send(msg)
  796. logger.info("[Email] Sent reply to %s (subject: %s)", to_addr, subject)
  797. return msg_id
  798. def _send_with_files(self, to_addr: str, body: str, files: list[tuple[Path, str]], *, lenient: bool,
  799. reply_to_msg_id: Optional[str] = None) -> str:
  800. """Send a reply with attachments; *lenient* logs-and-skips unattachable files instead of raising.
  801. An explicit *reply_to_msg_id* threads the mail like ``_send_email`` does (#10131)."""
  802. msg, msg_id, _ = self._new_reply(to_addr, body, reply_to_msg_id)
  803. for path, name in files:
  804. try:
  805. _attach_file(msg, path, name)
  806. except Exception as e:
  807. if not lenient:
  808. raise
  809. logger.warning("[Email] Failed to attach %s: %s", path, e)
  810. self._smtp_send(msg)
  811. return msg_id
  812. async def send_image(self, chat_id: str, image_url: str, caption: Optional[str] = None,
  813. reply_to: Optional[str] = None, metadata: Optional[dict[str, Any]] = None) -> SendResult:
  814. """Send an image URL as part of an email body (``metadata`` unused)."""
  815. return await self.send(chat_id, f"{caption or ''}\n\nImage: {image_url}".strip(), reply_to)
  816. async def send_multiple_images(self, chat_id: str, images: list[tuple[str, str]],
  817. metadata: Optional[dict[str, Any]] = None, human_delay: float = 0.0) -> SendResult:
  818. """One email per batch: local files attached, URL images linked in the body (no remote download); base-class fallback on failure."""
  819. if not images:
  820. return SendResult(success=False, error="no images to send")
  821. from urllib.parse import unquote as _unquote
  822. body_parts, local_paths = [], []
  823. for image_url, alt_text in images:
  824. if alt_text:
  825. body_parts.append(alt_text)
  826. if not image_url.startswith("file://"):
  827. body_parts.append(t("platform.email.image_line", url=image_url)) # parity with send_image
  828. elif Path(local_path := _unquote(image_url[7:])).exists():
  829. local_paths.append(local_path)
  830. else:
  831. logger.warning("[Email] Skipping missing image: %s", local_path)
  832. if not local_paths and not body_parts:
  833. return SendResult(success=False, error="no valid images in batch")
  834. try:
  835. message_id = await asyncio.get_running_loop().run_in_executor(None, self._send_email_with_attachments, chat_id, "\n\n".join(body_parts), local_paths)
  836. except Exception as e:
  837. logger.error("[Email] Multi-image send failed, falling back: %s", e, exc_info=True)
  838. return await super().send_multiple_images(chat_id, images, metadata, human_delay)
  839. return SendResult(success=True, message_id=message_id)
  840. def _send_email_with_attachments(self, to_addr: str, body: str, file_paths: list[str]) -> str:
  841. """Send an email with multiple file attachments via SMTP (unattachable files are skipped)."""
  842. msg_id = self._send_with_files(to_addr, body, [(Path(f), Path(f).name) for f in file_paths], lenient=True)
  843. logger.info("[Email] Sent multi-attachment email to %s (%d files)", to_addr, len(file_paths))
  844. return msg_id
  845. async def send_document(self, chat_id: str, file_path: str, caption: Optional[str] = None,
  846. file_name: Optional[str] = None, reply_to: Optional[str] = None, **kwargs) -> SendResult:
  847. """Send a file as an email attachment."""
  848. return await self._run_send(self._send_email_with_attachment, (chat_id, caption or "", file_path, file_name, reply_to),
  849. "[Email] Send document failed: %s")
  850. def _send_email_with_attachment(self, to_addr: str, body: str, file_path: str, file_name: Optional[str] = None,
  851. reply_to_msg_id: Optional[str] = None) -> str:
  852. """Send an email with a single file attachment via SMTP (raises if unattachable)."""
  853. return self._send_with_files(to_addr, body, [(Path(file_path), file_name or Path(file_path).name)], lenient=False,
  854. reply_to_msg_id=reply_to_msg_id)
  855. async def get_chat_info(self, chat_id: str) -> dict[str, Any]:
  856. """Return basic info about the email chat."""
  857. return {"name": chat_id, "type": "dm", "chat_id": chat_id, "subject": self._thread_context.get(chat_id, {}).get("subject", "")}
  858. # Plugin glue: register() exposes the platform via the registry; EMAIL_* env → PlatformConfig seeding stays in core.
  859. async def _standalone_send(pconfig, chat_id, message, *, thread_id=None, media_files=None, force_document=False):
  860. """Out-of-process Email delivery via SMTP (one-shot); standalone_sender_fn contract."""
  861. extra = getattr(pconfig, "extra", {}) or {}
  862. address, password = extra.get("address") or _get_secret("EMAIL_ADDRESS", ""), _get_secret("EMAIL_PASSWORD", "")
  863. smtp_host, smtp_port = extra.get("smtp_host") or _get_secret("EMAIL_SMTP_HOST", ""), _esecret_int("EMAIL_SMTP_PORT", 587)
  864. smtp_security = _normalize_security(_get_secret("EMAIL_SMTP_SECURITY", "") or extra.get("smtp_security"), default="tls" if smtp_port == 465 else "starttls")
  865. smtp_tls_verify = _esecret_bool("EMAIL_SMTP_TLS_VERIFY", is_truthy_value(extra.get("smtp_tls_verify"), default=True))
  866. if not all([address, password, smtp_host]):
  867. return send_error("Email not configured (EMAIL_ADDRESS, EMAIL_PASSWORD, EMAIL_SMTP_HOST required)")
  868. try:
  869. msg = MIMEText(message, "plain", "utf-8")
  870. for key, value in (("From", address), ("To", chat_id), ("Subject", t("platform.email.standalone_subject")), ("Date", formatdate(localtime=True))):
  871. msg[key] = value
  872. server = _open_smtp(smtp_host, smtp_port, smtp_security, _tls_context(smtp_tls_verify, smtp_host), smtplib.SMTP, smtplib.SMTP_SSL)
  873. server.login(address, password)
  874. server.send_message(msg)
  875. server.quit()
  876. return {"success": True, "platform": "email", "chat_id": chat_id}
  877. except Exception as e:
  878. try:
  879. from tools.send_message_tool import _error as _e
  880. return _e(f"Email send failed: {e}")
  881. except Exception:
  882. return send_error(f"Email send failed: {e}")
  883. def _is_connected(config) -> bool:
  884. """Connected when an address is configured (PlatformConfig.extra or EMAIL_ADDRESS)."""
  885. if (getattr(config, "extra", {}) or {}).get("address"):
  886. return True
  887. import hermes_cli.gateway as gateway_mod
  888. return bool((gateway_mod.get_env_value("EMAIL_ADDRESS") or "").strip())
  889. # System-prompt platform hint (PlatformEntry.platform_hint); cached-prompt bytes, keep stable.
  890. _PLATFORM_HINT = (
  891. "You are communicating via email. Write clear, well-structured responses suitable for email. Use "
  892. "plain text formatting (no markdown). Keep responses concise but complete. You can send file "
  893. "attachments — include MEDIA:/absolute/path/to/file in your response. The subject line is preserved "
  894. "for threading. Do not include greetings or sign-offs unless contextually appropriate."
  895. )
  896. _SEND_TARGET_RE = re.compile(r"^\s*[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\s*$")
  897. def register(ctx) -> None:
  898. """Plugin entry point — called by the Hermes plugin system."""
  899. from . import gateway_env # env -> GatewayConfig bridge (gateway.config_env)
  900. ctx.register_platform(
  901. target_ref_syntax_fn=regex_stripped_parser(_SEND_TARGET_RE),
  902. standalone_send=StandaloneSendPolicy(text_direct=True),
  903. name="email", label="Email", adapter_factory=EmailAdapter, check_fn=check_email_requirements, is_connected=_is_connected,
  904. dashboard_description="Talk to Hermes through an IMAP/SMTP mailbox.",
  905. docs_url="https://hermes-agent.nousresearch.com/docs/user-guide/messaging/",
  906. dashboard_env_vars=("EMAIL_ADDRESS", "EMAIL_PASSWORD", "EMAIL_IMAP_HOST", "EMAIL_SMTP_HOST"),
  907. dashboard_required_env=("EMAIL_ADDRESS", "EMAIL_PASSWORD", "EMAIL_IMAP_HOST", "EMAIL_SMTP_HOST"),
  908. required_env=["EMAIL_ADDRESS", "EMAIL_PASSWORD", "EMAIL_SMTP_HOST"],
  909. install_hint="Email uses the Python stdlib (smtplib/imaplib) — no extra deps", cron_deliver_env_var="EMAIL_HOME_ADDRESS", standalone_sender_fn=_standalone_send,
  910. max_message_length=50_000, pii_safe=True, emoji="📧", allow_update_command=True,
  911. platform_hint=_PLATFORM_HINT,
  912. display_tier="minimal",
  913. env_overrides=gateway_env.ENV_OVERRIDES,
  914. env_enable_credentials=gateway_env.ENV_ENABLE_CREDENTIALS, **gateway_env.AUTHZ,
  915. )