| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539 |
- """Scratch-dir retention: idle detection plus the reaping an idle tree needs before it goes.
- Deleting an idle ``cache/scratch/<entry>`` is not enough on its own: a lane's e2e run leaves
- headless browsers whose cwd was inside the tree (they survived for days with a ``(deleted)``
- cwd), and a repo whose linked worktree lived in the tree keeps a dangling registration until
- someone runs ``git worktree prune``. Both are reaped here, right before ``rmtree``.
- A process Hermes itself started is never an orphan, though: an idle entry a registered live
- process still works inside is kept, not reaped (#132401). Ownership comes from the two
- registries Hermes writes, not from a marker file a lane must remember to drop.
- """
- from __future__ import annotations
- import contextlib
- import json
- import logging
- import os
- import shutil
- import subprocess
- import time
- from logging.handlers import RotatingFileHandler
- from pathlib import Path
- import hermes_constants_scratch_stuck as stuck
- logger = logging.getLogger(__name__)
- # How long a TERMed process gets before KILL; browsers exit well within this.
- _REAP_GRACE_SECONDS = 3.0
- # ``.git`` files (linked worktrees) are looked for this deep; lanes nest repo/tree/subtree.
- _GIT_FILE_MAX_DEPTH = 4
- # logs/scratch-prune.log: one line per entry deleted, per process signalled, and per
- # entry rescued or vanished mid-prune (#132401). Same fixed rotation as tool_calls.log.
- _PRUNE_LOG_MAX_BYTES = 5 * 1024 * 1024
- _PRUNE_LOG_BACKUPS = 3
- # Paths and process names go in with %r: a control character (a newline in a legal POSIX name)
- # is written escaped, so it cannot split a record or forge one.
- class _PruneLogHandler(RotatingFileHandler):
- """Rollover that tolerates a concurrent prune of the same home rotating first: its rename
- of ``scratch-prune.log`` can land between this handler's existence check and its own
- rename. The rotation is then already done, so the record goes to the new file instead of
- being dropped with a FileNotFoundError."""
- def doRollover(self) -> None:
- with contextlib.suppress(FileNotFoundError):
- super().doRollover()
- def _open_prune_log(log_file: Path | None) -> logging.Logger:
- """Logger for one prune's records, with its own RotatingFileHandler on *log_file*, so a
- prune that runs at boot (before ``setup_logging()``) still leaves them on disk. Records also
- propagate through this module's logger to whatever logging is set up.
- One unregistered logger per prune, closed when it ends: no process holds the file open
- between prunes (an open handle blocks rollover on Windows), and prunes of two homes never
- share a handler. Without a *log_file* (test-runner temp roots) this module's logger is used.
- """
- if log_file is None:
- return logger
- try:
- log_file.parent.mkdir(parents=True, exist_ok=True)
- except OSError as exc:
- logger.warning("scratch prune: cannot create %s: %s", log_file.parent, exc)
- return logger
- audit = logging.Logger(logger.name, logging.INFO)
- audit.parent = logger
- # backslashreplace: a legal POSIX name that is not UTF-8 is written escaped instead of
- # dropping the whole record.
- handler = _PruneLogHandler(
- log_file, maxBytes=_PRUNE_LOG_MAX_BYTES, backupCount=_PRUNE_LOG_BACKUPS, encoding="utf-8",
- errors="backslashreplace", delay=True,
- )
- handler.setFormatter(logging.Formatter("%(asctime)s %(message)s"))
- audit.addHandler(handler)
- return audit
- def _tree_bytes(path: Path) -> int:
- """Apparent size of *path* and everything under it, symlinks not followed."""
- try:
- total = os.lstat(path).st_size
- except OSError:
- return 0
- stack = [str(path)] if path.is_dir() and not path.is_symlink() else []
- while stack:
- try:
- with os.scandir(stack.pop()) as it:
- for child in it:
- try:
- total += child.stat(follow_symlinks=False).st_size
- except OSError:
- continue
- if child.is_dir(follow_symlinks=False):
- stack.append(child.path)
- except OSError:
- continue
- return total
- _TOUCHED = "touched"
- _SCAN_FAILED = "scan-failed"
- _IDLE = "idle"
- def subtree_touch_state(path: Path, cutoff: float) -> str:
- """Tri-state form of :func:`subtree_touched_since`: ``touched`` (recent write seen),
- ``scan-failed`` (unreadable — idle cannot be established, so the entry is kept, but
- a failed scan is not activity and is never reported as such), or ``idle``.
- Same walk, same stop-at-first-recent shortcut, same symlink rules. A hardlinked file
- (``st_nlink > 1``) does not count either: its mtime is the inode's, and
- ``git clone --local`` hardlinks packs that git freshens from the source repo."""
- return _scan_subtree(path, cutoff)[0]
- def _scan_subtree(path: Path, cutoff: float) -> tuple[str, str | None]:
- """``(state, denied_dir)``: *denied_dir* is the directory below *path* that refused
- listing with EACCES/EPERM when that is why the scan failed (else None)."""
- try:
- if os.lstat(path).st_mtime >= cutoff:
- return _TOUCHED, None
- if not path.is_dir() or path.is_symlink():
- return _IDLE, None
- except OSError:
- return _SCAN_FAILED, None
- stack = [str(path)]
- while stack:
- current = stack.pop()
- try:
- with os.scandir(current) as it:
- for child in it:
- try:
- st = child.stat(follow_symlinks=False)
- except OSError as exc:
- return _SCAN_FAILED, current if stuck.is_permission_error(exc) else None
- if st.st_mtime >= cutoff and not (st.st_nlink > 1 and child.is_file(follow_symlinks=False)):
- return _TOUCHED, None
- if child.is_dir(follow_symlinks=False):
- stack.append(child.path)
- except OSError as exc:
- return _SCAN_FAILED, current if stuck.is_permission_error(exc) else None
- return _IDLE, None
- def subtree_touched_since(path: Path, cutoff: float) -> bool:
- """True when *path* or anything beneath it has an mtime at or after *cutoff*.
- Stops at the first recent entry, so a live tree costs one hit and only a truly idle
- tree pays for the full walk (once, right before it is deleted). Symlinks are never
- followed: a link into the repo would make the target's activity keep the entry alive.
- An unreadable entry is kept: an incomplete scan cannot establish that it is idle —
- so True here means "touched **or** uninspectable", and callers that need to tell the
- two apart (the mid-prune audit) use :func:`subtree_touch_state` instead.
- """
- return subtree_touch_state(path, cutoff) != _IDLE
- def _under(path: str, root: str) -> bool:
- return path == root or path.startswith(root.rstrip(os.sep) + os.sep)
- # The ancestry of the Hermes command that launched a detached pass (hermes_constants_scratch_detach),
- # as ``(pid, create_time)`` pairs the launcher computed before it spawned the pass: the pass is
- # reparented once that command exits, so its own ancestry no longer reaches the user's shell.
- # ``None``: the launcher could not compute it, so the pass signals nothing and keeps every entry
- # a live process works in.
- _launcher_lineage: frozenset[tuple[int, float]] | None = frozenset()
- _HAS_PROC = os.path.isdir("/proc/self")
- def _birth_stamp(proc) -> float:
- """A start stamp for *proc* that a clock step cannot move. Linux: ``starttime`` from
- ``/proc/<pid>/stat`` (clock ticks since boot; psutil's ``create_time()`` adds a wall-clock
- boot time, so an NTP/VM/resume step shifts it by whole seconds). Elsewhere:
- :func:`hermes_cli.process_identity.process_birth` (the unadjusted kernel start on macOS).
- On a ``/proc`` host a failed read raises: falling back to a wall-clock value would compare
- an epoch against recorded ticks and disown a live process."""
- if _HAS_PROC:
- # Bytes, not text: comm is cut at 15 bytes and can end mid-UTF-8 sequence. After the
- # last ")" (comm may hold spaces or ")") fields start at #3, so #22 is index 19.
- stat = Path(f"/proc/{proc.pid}/stat").read_bytes()
- return float(int(stat.rsplit(b")", 1)[1].split()[19]))
- from hermes_cli.process_identity import process_birth
- return process_birth(proc)
- def _same_birth(proc, started: float) -> bool:
- """Exact on Linux (integer ticks); a 2 s tolerance where the stamp is a float epoch."""
- now = _birth_stamp(proc)
- return now == started if _HAS_PROC else abs(now - started) < 2.0
- def lineage_with_start_times() -> list[tuple[int, float]] | None:
- """This process and its ancestors with their create times, the set :func:`_own_lineage`
- computes in-process, in a form that stays checkable after they exit. ``None`` when it
- cannot be computed completely (no psutil, a process that refuses inspection)."""
- try:
- import psutil
- except ImportError:
- return None
- chain: dict[int, float] = {}
- try:
- proc = psutil.Process()
- while proc is not None and proc.pid not in chain:
- chain[proc.pid] = _birth_stamp(proc)
- proc = proc.parent()
- except (psutil.Error, OSError, IndexError, ValueError):
- return None
- return list(chain.items())
- def _own_lineage() -> set[int] | None:
- """This process and its ancestors: never reap the shell that is running the prune. A
- detached pass adds its launcher's ancestors that still run under the same create time (a
- reused pid is not spared), and returns ``None`` (spare every process) without them."""
- import psutil
- if _launcher_lineage is None:
- return None
- pids: set[int] = set()
- try:
- proc = psutil.Process()
- while proc is not None and proc.pid not in pids:
- pids.add(proc.pid)
- proc = proc.parent()
- except (psutil.Error, OSError):
- pass
- for pid, started in _launcher_lineage:
- try:
- if _same_birth(psutil.Process(pid), started):
- pids.add(pid)
- except psutil.NoSuchProcess:
- continue
- except (psutil.Error, OSError, IndexError, ValueError):
- pids.add(pid) # unreadable is not proof of a reused pid: spare it
- return pids
- def _registered_live_pids(home: Path) -> set[int]:
- """Live pids Hermes registered for *home*: the machine spawn ledger (every long-lived entry
- point and MCP helper, ``hermes_cli.process_identity``) and the background-process checkpoint
- (``tools.process_registry``, ``processes.json``). A recorded start time that no longer
- matches is a reused pid, not ownership."""
- import psutil
- from hermes_cli.process_identity import entry_still_running, spawner_is_dead
- from hermes_constants import get_default_hermes_root
- root = get_default_hermes_root(home=home)
- sources = [(root / "spawn-ledger.json", "create_time")]
- sources += [(base / "processes.json", "host_start_time") for base in {home, root}]
- pids: set[int] = set()
- for path, stamp_key in sources:
- try:
- entries = json.loads(path.read_text(encoding="utf-8-sig"))
- except (OSError, ValueError):
- continue
- for entry in entries if isinstance(entries, list) else []:
- pid, stamp = (entry.get("pid"), entry.get(stamp_key)) if isinstance(entry, dict) else (None, None)
- if not isinstance(pid, int) or pid <= 0 or entry.get("pid_scope", "host") != "host":
- continue
- # An MCP helper whose recorded spawner is provably gone is an orphan, not owned: the
- # rule reap_orphaned_mcp_helpers applies. Long-lived entry points (gateway, serve) that
- # outlive their launcher stay owned; an unprovable spawner keeps ownership.
- if entry.get("purpose") == "mcp-helper" and spawner_is_dead(entry) is True:
- continue
- try:
- if stamp_key == "create_time":
- alive = entry_still_running(entry) # boot ticks survive a clock step
- elif stamp is None:
- alive = psutil.pid_exists(pid)
- else:
- from gateway.status import get_process_start_time
- alive = get_process_start_time(pid) == stamp
- except (psutil.Error, OSError, TypeError, ValueError):
- continue
- if alive:
- pids.add(pid)
- return pids
- def _owned_by_hermes(proc, registered: set[int]) -> bool:
- """Registered itself, or a descendant of a live registered process (a background shell's
- child). An orphan whose spawning shell is gone was reparented away from that lineage, so
- it is not owned and is still reaped."""
- import psutil
- try:
- # pid 1 is excluded as an ancestor: a Hermes running as a container init adopts every orphan.
- return proc.pid in registered or any(p.pid in registered and p.pid != 1 for p in proc.parents())
- except (psutil.Error, OSError):
- return False
- def reap_processes_rooted_in(scratch_root: Path, doomed: list[Path], audit: logging.Logger = logger) -> int:
- """TERM (then KILL) same-user processes whose cwd is inside an entry about to be
- pruned, or is a path under *scratch_root* that no longer exists. Returns the count.
- Only the cwd is consulted: a process merely holding a file open inside scratch (an
- editor, a log tail) is not ours to kill, but one *living* in a directory we are
- about to delete, or in one already gone, has nothing left to run for.
- A process Hermes owns (:func:`_owned_by_hermes`) is never signalled; each doomed entry it
- works inside is dropped from *doomed* in place and recorded as kept.
- """
- import psutil
- root = os.path.realpath(str(scratch_root))
- targets = [os.path.realpath(str(p)) for p in doomed]
- skip = _own_lineage()
- uid = os.getuid() if hasattr(os, "getuid") else None
- victims: list[tuple[psutil.Process, str]] = []
- # The pruner's own lineage is never signalled, but its cwd still keeps an entry: a live
- # session that launched this Hermes command may be working inside it.
- lineage: list[tuple[psutil.Process, str]] = []
- for proc in psutil.process_iter(["pid"]):
- try:
- if uid is not None and proc.uids().real != uid:
- continue
- cwd = proc.cwd()
- except (psutil.Error, OSError):
- continue
- if not cwd:
- continue
- deleted = cwd.endswith(" (deleted)")
- cwd_path = cwd[: -len(" (deleted)")] if deleted else cwd
- if not _under(cwd_path, root):
- continue
- if deleted or not os.path.exists(cwd_path) or any(_under(cwd_path, t) for t in targets):
- (lineage if skip is None or proc.pid in skip else victims).append((proc, cwd_path))
- # The owning home comes from the scratch path as configured, not its resolved target:
- # a cache/scratch symlinked onto another disk still belongs to <home>.
- registered = _registered_live_pids(Path(scratch_root).absolute().parent.parent) if victims else set()
- owned = lineage + [(proc, cwd_path) for proc, cwd_path in victims if _owned_by_hermes(proc, registered)]
- for proc, cwd_path in owned:
- for entry in [e for e in doomed if _under(cwd_path, os.path.realpath(str(e)))]:
- doomed.remove(entry)
- try:
- name = proc.name()
- except (psutil.Error, OSError):
- name = "?"
- audit.info("scratch prune: kept %r — live Hermes process pid=%d (%r) works inside",
- os.fspath(entry), proc.pid, name)
- # Re-filter against the shrunk *doomed*: an orphan in a kept entry is spared with it (the
- # reap only exists to clear the way for rmtree); one whose cwd is already gone still goes.
- victims = [(proc, cwd_path) for proc, cwd_path in victims if (proc, cwd_path) not in owned
- and (not os.path.exists(cwd_path) or any(_under(cwd_path, os.path.realpath(str(e))) for e in doomed))]
- if not victims:
- return 0
- for proc, cwd_path in victims:
- try:
- name = proc.name()
- proc.terminate()
- except (psutil.Error, OSError):
- continue
- audit.info("scratch prune: sent TERM pid=%d name=%r cwd=%r", proc.pid, name, cwd_path)
- _, alive = psutil.wait_procs([proc for proc, _ in victims], timeout=_REAP_GRACE_SECONDS)
- for proc in alive:
- try:
- # A zombie already exited; only its parent has not collected it yet.
- if proc.status() == psutil.STATUS_ZOMBIE:
- continue
- proc.kill()
- except (psutil.Error, OSError):
- continue
- audit.info("scratch prune: sent KILL pid=%d (still running after TERM)", proc.pid)
- logger.info("scratch prune: reaped %d process(es) rooted in pruned entries", len(victims))
- return len(victims)
- def _linked_worktree_repos(entry: Path) -> set[str]:
- """Repos whose linked worktrees live inside *entry* (``.git`` FILES, ``gitdir: <repo>/.git/worktrees/<n>``)."""
- repos: set[str] = set()
- stack = [(str(entry), 0)]
- while stack:
- current, depth = stack.pop()
- try:
- with os.scandir(current) as it:
- for child in it:
- if child.name == ".git" and child.is_file(follow_symlinks=False):
- try:
- line = Path(child.path).read_text(encoding="utf-8-sig", errors="replace").strip()
- except OSError:
- continue
- if line.startswith("gitdir:"):
- gitdir = Path(line[len("gitdir:"):].strip())
- # <repo>/.git/worktrees/<name> -> <repo>
- if gitdir.parent.name == "worktrees" and gitdir.parent.parent.name == ".git":
- repos.add(str(gitdir.parent.parent.parent))
- elif child.is_dir(follow_symlinks=False) and depth < _GIT_FILE_MAX_DEPTH \
- and child.name not in ("node_modules", ".venv", "venv"):
- stack.append((child.path, depth + 1))
- except OSError:
- continue
- return repos
- def release_git_worktrees(repos: set[str]) -> None:
- """``git worktree prune`` in each repo: drops registrations whose tree we just deleted."""
- for repo in sorted(repos):
- if not os.path.isdir(repo):
- continue
- try:
- subprocess.run(
- ["git", "-C", repo, "worktree", "prune"],
- stdin=subprocess.DEVNULL, capture_output=True, text=True, encoding="utf-8", errors="replace",
- timeout=15, check=False,
- )
- except (OSError, subprocess.SubprocessError) as exc:
- logger.debug("git worktree prune in %s failed: %s", repo, exc)
- def prune_idle_entries(
- root: Path, max_idle_hours: float, skip_names: frozenset[str], log_file: Path | None = None,
- ) -> int:
- """Delete top-level entries of *root* with no write anywhere in their subtree for
- *max_idle_hours*, reaping processes and worktree registrations rooted in them first.
- Each removal, each signalled process and each entry rescued mid-prune (touched
- again after selection) is recorded in *log_file*. Returns the count actually
- removed."""
- audit = _open_prune_log(log_file)
- try:
- return _prune_idle_entries(root, max_idle_hours, skip_names, audit)
- finally:
- if audit is not logger:
- for handler in audit.handlers:
- handler.close()
- def _prune_idle_entries(
- root: Path, max_idle_hours: float, skip_names: frozenset[str], audit: logging.Logger,
- ) -> int:
- cutoff = time.time() - max_idle_hours * 3600
- # Entries already known to hold files this user cannot read or delete are skipped before
- # any walk, until that is fixed (hermes_constants_scratch_stuck).
- skip_names = skip_names | stuck.load_skips(root) | {stuck.STUCK_STATE_NAME, stuck.STUCK_LOCK_NAME}
- try:
- entries = [e for e in root.iterdir() if e.name not in skip_names]
- except OSError:
- return 0
- doomed = []
- for e in entries:
- state, denied_dir = _scan_subtree(e, cutoff)
- if state == _IDLE:
- doomed.append(e)
- elif denied_dir is not None and stuck.tracking_supported():
- # A dir this user may not list (root-owned 0700) fails every idle scan: kept
- # (fail-closed) as before, but reported once and no longer re-scanned hourly.
- stuck.record(root, e, [denied_dir], audit)
- # Runs even with nothing to delete: orphans whose cwd was removed by an earlier pass
- # (or by hand) are found by the deleted-cwd rule, not by membership in ``doomed``.
- try:
- reap_processes_rooted_in(root, doomed, audit)
- # health: allow BLE001 -- fail-closed boundary: any failure leaves who works inside unknown; audited below
- except Exception as exc: # psutil missing or restricted host
- audit.info("scratch prune: process check failed (%s: %r), nothing deleted this pass",
- type(exc).__name__, str(exc))
- return 0
- if not doomed:
- return 0
- repos: set[str] = set()
- removed = 0
- for entry in doomed:
- # Worktree scan first: a registration whose tree an earlier pass (or hand)
- # removed is stale even in a rescued entry, and the re-check below sits as
- # close to the delete as the loop allows.
- if entry.is_dir() and not entry.is_symlink():
- repos |= _linked_worktree_repos(entry)
- # Last-moment re-validation (#132401 C1/F1): ``doomed`` is a snapshot from
- # before the reap ran, and a writer whose cwd is outside scratch — invisible
- # to the reap — can land fresh work in that window. The snapshot is a
- # candidate list, never a verdict: anything touched since selection is
- # rescued here, not deleted. A re-scan that cannot read the entry is also
- # fail-closed (the entry is kept), but a failed scan is not activity — the
- # audit names the failed re-scan instead of inventing a touch that did not
- # happen, so operators can tell an uninspectable candidate from verified
- # recent activity (P3 on #134173).
- # Size is collected before the re-check, not after it: a full-tree walk between
- # the re-check and rmtree reopens the window for a write landing during that
- # walk to be destroyed (review F6 on #139850).
- size = _tree_bytes(entry)
- state = subtree_touch_state(entry, cutoff)
- if state != _IDLE:
- # Only a missing path is a vanish: a stat that raises PermissionError means
- # the entry is there but unreadable, and reporting it as vanished would be
- # the same invented reason the audit exists to prevent.
- try:
- os.lstat(entry)
- entry_gone = False
- except FileNotFoundError:
- entry_gone = True
- except OSError:
- entry_gone = False
- if entry_gone:
- audit.info("scratch prune: entry %r vanished since selection", os.fspath(entry))
- elif state == _TOUCHED:
- audit.info(
- "scratch prune: rescued %r — touched since selection, kept",
- os.fspath(entry),
- )
- else:
- audit.info(
- "scratch prune: kept %r — activity re-scan failed, left untouched",
- os.fspath(entry),
- )
- continue
- denied: list[str] = []
- try:
- if entry.is_dir() and not entry.is_symlink():
- denied = stuck.rmtree_collecting_permission_errors(entry)
- else:
- entry.unlink()
- except OSError as exc:
- audit.info("scratch prune: could not remove %r: %s", os.fspath(entry), exc)
- continue
- # rmtree ignores errors, so only a path that is really gone counts as removed.
- # Only an explicit lstat proves absence: lexists() swallows OSError from lstat,
- # so an entry that stayed (rmtree left residue) but cannot be inspected would be
- # counted and recorded as removed — the same invented reason the vanish rule
- # above refuses to invent (P2 on #134173).
- try:
- os.lstat(entry)
- if denied and stuck.tracking_supported():
- stuck.record(root, entry, denied, audit)
- else:
- audit.info("scratch prune: could not fully remove %r", os.fspath(entry))
- continue
- except FileNotFoundError:
- pass
- except OSError:
- audit.info("scratch prune: could not confirm removal of %r", os.fspath(entry))
- continue
- audit.info("scratch prune: removed %r (%d bytes)", os.fspath(entry), size)
- removed += 1
- release_git_worktrees(repos)
- return removed
|