| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464 |
- """hermes.provider_setup.count: where connecting a model provider starts, succeeds, fails or is walked away from.
- A flow is ``started`` once the user has picked a provider. Its end is recorded by the surface that
- ran it (``completed`` / ``failed`` + a closed failure class). A flow nobody finished cannot record
- itself, so ``started`` also drops a small marker under the owning profile's store dir (the
- process-exit pattern): the finisher claims it by rename, and a marker whose process is gone (or that
- has been pending longer than any real flow takes) is reported ``abandoned`` by the next setup start
- or Hermes start in that profile. Whoever claims the marker records, so each flow ends exactly once.
- Only the catalog provider name leaves (custom endpoints read ``custom``); never a key, token, base
- URL or error text. Rows are saved synchronously: a setup killed right after ``started`` must still
- have left its row, and none of this runs on a hot path.
- """
- from __future__ import annotations
- import contextlib
- import json
- import logging
- import os
- import threading
- import time
- from dataclasses import dataclass
- from pathlib import Path
- from typing import Any, Iterator
- logger = logging.getLogger(__name__)
- MARKER_DIRNAME = "provider_setup_markers"
- # Device codes expire in ~15 minutes; a flow still pending an hour later was left open and forgotten.
- STALE_AFTER_S = 3600
- # Picker rows that are not a provider (menus, removal) never start a setup flow.
- _NOT_A_PROVIDER = frozenset({"aux-config", "cancel", "reasoning", "remove-custom"})
- _AUTH_ERROR_TYPES = frozenset({"AuthError", "SignInCopyError"})
- _OAUTH_SESSION_KEY = "_metrics_setup_flow"
- _OAUTH_FAILURE_KEY = "_metrics_setup_failure"
- # Ends that are not failures: the user stopped it, or let the sign-in code run out.
- _NOT_FAILURES = frozenset({"cancelled", "expired"})
- # Tokens other tools read too (gh, the Hugging Face Hub): saving one is not connecting that provider.
- _SHARED_TOKENS = frozenset({"GH_TOKEN", "GITHUB_TOKEN", "HF_TOKEN"})
- # OAuth session status (+ reason) at its end -> (event, failure_class).
- _OAUTH_ENDINGS = {
- "approved": ("completed", "none"),
- "cancelled": ("abandoned", "none"),
- "denied": ("failed", "auth"),
- "expired": ("abandoned", "none"),
- }
- _OAUTH_REASONS = {
- "user_declined": ("abandoned", "none"),
- "superseded": ("abandoned", "none"),
- "timeout": ("abandoned", "none"),
- "account_retired": ("failed", "auth"),
- "anon_unreachable": ("failed", "network"),
- }
- @dataclass
- class SetupFlow:
- home: str
- surface: str
- provider: str # already the catalog metric name
- marker: Path
- def markers_dir(home: Path) -> Path:
- return home / "telemetry" / "shared_metrics" / MARKER_DIRNAME
- @contextlib.contextmanager
- def _bound(hermes_home: Any) -> Iterator[None]:
- token = None
- if hermes_home:
- from hermes_constants import set_hermes_home_override
- token = set_hermes_home_override(str(hermes_home))
- try:
- yield
- finally:
- if token is not None:
- from hermes_constants import reset_hermes_home_override
- reset_hermes_home_override(token)
- def collection_enabled(hermes_home: Any = None) -> bool:
- """The owning profile's shared-metrics gate (cheap; lets async callers skip the thread hop)."""
- try:
- with _bound(hermes_home):
- from .relay_shared_metrics import enabled
- return bool(enabled())
- except Exception:
- return False
- def provider_setup_fields(*, surface: Any, provider: Any, event: Any, failure_class: Any = "none") -> dict[str, str] | None:
- from . import shared_metrics_contract as contract
- from .shared_metrics_catalog import provider_metric_name
- if surface not in contract.PROVIDER_SETUP_SURFACES or event not in contract.PROVIDER_SETUP_EVENTS:
- return None
- failure = "none"
- if event == "failed" and failure_class in _NOT_FAILURES: # a cancel or a lapsed code is a walk-away
- event = "abandoned"
- elif event == "failed":
- failure = failure_class if failure_class in contract.PROVIDER_SETUP_FAILURE_CLASSES - {"none"} else "other"
- return {"event": event, "failure_class": failure, "provider": provider_metric_name(provider), "surface": surface}
- def setup_failure_class(exc: BaseException) -> str:
- """Closed class for an exception that ended a setup flow; inert (type checks only). ``cancelled``
- and ``expired`` (the sign-in code ran out before approval) are recorded ``abandoned``."""
- from hermes_cli.auth_error_copy import is_cancelled, is_device_code_expired, is_network_error
- names = {cls.__name__ for cls in type(exc).__mro__}
- if isinstance(getattr(exc, "setup_failure_class", None), str): # classified where it was raised
- return exc.setup_failure_class
- # Esc in the setup menus, or consent declined on the provider's page.
- if is_cancelled(exc) or "_SetupCancelled" in names or getattr(exc, "oauth_error_code", "") == "access_denied":
- return "cancelled"
- # Before the expiry check: a code that ran out while the service kept failing is an outage.
- if is_network_error(exc) or (exc.__cause__ is not None and is_network_error(exc.__cause__)):
- return "network"
- if is_device_code_expired(exc):
- return "expired"
- if names & _AUTH_ERROR_TYPES:
- return "auth"
- return "other"
- def _save_row(fields: dict[str, str] | None) -> bool:
- """Save one row and wait for the store; a row the contract cannot build is settled as is."""
- if fields is None:
- return True
- from .shared_metrics_contract import PROVIDER_SETUP_MARK
- from .shared_metrics_events import emit_saved
- return emit_saved([(PROVIDER_SETUP_MARK, fields)]) == 1
- def begin_provider_setup(surface: str, provider: Any, *, hermes_home: Any = None) -> SetupFlow | None:
- """Record ``started`` and drop the pending marker; None (no files, no rows) when collection is off."""
- try:
- with _bound(hermes_home):
- from .relay_shared_metrics import enabled
- if not provider or provider in _NOT_A_PROVIDER or not enabled():
- return None
- fields = provider_setup_fields(surface=surface, provider=provider, event="started")
- if fields is None:
- return None
- from gateway.status import get_process_start_time
- from hermes_constants import get_hermes_home
- home = get_hermes_home()
- report_abandoned_setups(home)
- directory = markers_dir(home)
- directory.mkdir(parents=True, exist_ok=True)
- pid = os.getpid()
- marker = directory / f"{surface}-{pid}-{time.time_ns()}.json"
- record = {
- "pid": pid, "start_time": get_process_start_time(pid), "started_at": time.time(),
- "surface": surface, "provider": fields["provider"],
- }
- tmp = marker.with_name(f".{marker.name}.tmp")
- tmp.write_text(json.dumps(record), encoding="utf-8")
- os.replace(tmp, marker)
- _save_row(fields)
- return SetupFlow(home=str(home), surface=surface, provider=fields["provider"], marker=marker)
- except Exception:
- logger.debug("Provider setup start not recorded", exc_info=True)
- return None
- def finish_provider_setup(flow: SetupFlow | None, event: str, failure_class: str = "none") -> None:
- """Record the flow's end once: only if its marker is still unclaimed (never raises)."""
- if flow is None:
- return
- try:
- from .shared_metrics_process import settle_claim
- claimed = flow.marker.with_name(f"{flow.marker.name}.{os.getpid()}.reporting")
- try:
- os.replace(flow.marker, claimed)
- except OSError: # already reported (abandoned) by another start
- return
- with _bound(flow.home):
- fields = provider_setup_fields(
- surface=flow.surface, provider=flow.provider, event=event, failure_class=failure_class,
- )
- settle_claim(claimed, flow.marker, _save_row(fields))
- except Exception:
- logger.debug("Provider setup end not recorded", exc_info=True)
- def _reportable(record: Any) -> bool:
- from gateway.status import runtime_status_pid_is_live
- from .shared_metrics_contract import _non_negative_number
- if not isinstance(record, dict):
- return True
- started = _non_negative_number(record.get("started_at"))
- if started is None or time.time() - started > STALE_AFTER_S:
- return True
- return not runtime_status_pid_is_live(record)
- def report_abandoned_setups(home: Path) -> None:
- """Report every pending marker whose flow can no longer finish as ``abandoned``.
- Callers bind ``home`` (the scan runs at a setup start or on the process-exit reporter thread).
- """
- from .shared_metrics_process import _claimer_alive, settle_claim
- directory = markers_dir(home)
- if not directory.is_dir():
- return
- for path in sorted(directory.iterdir()):
- if path.name.startswith(".") or ".json" not in path.name:
- continue
- if path.name.endswith(".reporting") and _claimer_alive(path):
- continue
- try:
- record = json.loads(path.read_text(encoding="utf-8-sig"))
- except (OSError, ValueError):
- record = None
- if not path.name.endswith(".reporting") and not _reportable(record):
- continue
- original = path.with_name(path.name.split(".json")[0] + ".json")
- claimed = original.with_name(f"{original.name}.{os.getpid()}.reporting")
- try:
- os.replace(path, claimed)
- except OSError: # a concurrent reporter or the finisher won
- continue
- fields = provider_setup_fields(
- surface=record.get("surface"), provider=record.get("provider"), event="abandoned",
- ) if isinstance(record, dict) else None
- settle_claim(claimed, original, _save_row(fields))
- def record_provider_setup_done(surface: str, provider: Any, *, hermes_home: Any = None, background: bool = False) -> None:
- """A setup that starts and lands in one action (an API key saved from a form): both rows at once.
- ``background`` keeps a request handler from waiting on a cold metrics runtime; the owning home is
- captured here, since a thread does not inherit the profile binding."""
- try:
- with _bound(hermes_home):
- from .relay_shared_metrics import enabled
- if not provider or not enabled():
- return
- from hermes_constants import get_hermes_home
- home = str(get_hermes_home())
- def run() -> None:
- finish_provider_setup(begin_provider_setup(surface, provider, hermes_home=home), "completed")
- if background:
- threading.Thread(target=run, name="hermes-setup-metrics", daemon=True).start()
- else:
- run()
- except Exception:
- logger.debug("Provider setup not recorded", exc_info=True)
- def provider_for_api_key_env(env_var: Any, *, connecting: bool = False) -> str | None:
- """The shipped provider whose own API key lives in ``env_var``; None for any other variable. A bare
- key save cannot tell connecting a provider from configuring a tool, so unless the caller says it is
- ``connecting`` one, ecosystem tokens and keys a tool's settings panel (TTS, STT, image...) also asks
- for are None too."""
- from hermes_cli.auth import PROVIDER_REGISTRY
- from hermes_cli.tools_config import TOOL_CATEGORIES
- if env_var == "OPENROUTER_API_KEY": # the aggregator is not a registry entry
- return "openrouter"
- if not connecting and (env_var in _SHARED_TOKENS or any(
- env_var == entry.get("key") for category in TOOL_CATEGORIES.values()
- for row in category.get("providers", ()) for entry in row.get("env_vars") or ()
- )):
- return None
- return next((slug for slug, pconfig in PROVIDER_REGISTRY.items()
- if env_var in (getattr(pconfig, "api_key_env_vars", None) or ())), None)
- def record_api_key_saved(env_var: Any, value: Any, previous: Any, surface: str, *, connecting: bool = False) -> None:
- """Count a provider API key saved from a settings form: only a new or changed non-empty value (a
- clear or a same-value re-save connects nothing; other variables are not setups)."""
- try:
- from .relay_shared_metrics import enabled
- if not isinstance(value, str) or not value.strip() or value == previous:
- return
- if enabled() and (provider := provider_for_api_key_env(env_var, connecting=connecting)):
- record_provider_setup_done(surface, provider, background=True)
- except Exception:
- logger.debug("API key setup not recorded", exc_info=True)
- # ---- CLI flows (`hermes setup`, `hermes model`, first-run chat) ------------------------------
- _cli = threading.local()
- def note_provider_setup_saved() -> None:
- """A CLI flow persisted the provider/model choice (called by the shared save helpers)."""
- hints = getattr(_cli, "hints", None)
- if hints is not None:
- hints["saved"] = True
- def note_provider_setup_failure(failure_class: str) -> None:
- """A CLI flow reported a failure it then returned from instead of raising (first one wins)."""
- hints = getattr(_cli, "hints", None)
- if hints is not None and hints.get("failure") is None:
- hints["failure"] = failure_class
- def note_sign_in_failure(exc: BaseException) -> None:
- with contextlib.suppress(Exception):
- note_provider_setup_failure(setup_failure_class(exc))
- def _model_route() -> tuple[Any, ...]:
- from hermes_cli.config import load_config
- model = load_config().get("model")
- if not isinstance(model, dict):
- return (model,)
- return tuple(model.get(key) for key in ("provider", "default", "base_url"))
- @contextlib.contextmanager
- def provider_setup_surface(surface: str) -> Iterator[None]:
- """Name the surface whose provider picker runs inside (``hermes model``, the setup wizard, the
- first-run chat prompt). The picker is shared, so only these entry points know which one it is."""
- previous = getattr(_cli, "surface", None)
- _cli.surface = surface
- try:
- yield
- except BaseException as exc:
- if not _is_go_back(exc): # a Back replays the wizard section: its flow may resume there
- _close_backed_out()
- raise
- else:
- _close_backed_out()
- finally:
- _cli.surface = previous
- def _is_go_back(exc: BaseException) -> bool:
- return any(cls.__name__ == "_SetupGoBack" for cls in type(exc).__mro__)
- def _close_backed_out() -> None:
- """The user went Back out of a provider flow and never resumed it: it ends ``abandoned``."""
- pending, _cli.backed_out = getattr(_cli, "backed_out", None), None
- if pending is not None:
- finish_provider_setup(pending[0], "abandoned")
- def _resumes(flow: SetupFlow, surface: Any, provider: Any) -> bool:
- try:
- from .shared_metrics_catalog import provider_metric_name
- return provider not in _NOT_A_PROVIDER and (flow.surface, flow.provider) == (surface, provider_metric_name(provider))
- except Exception:
- return False
- @contextlib.contextmanager
- def cli_provider_setup(provider: Any) -> Iterator[None]:
- """Track one CLI provider flow under the entry point's surface. Without one (fallback picking,
- tests driving the picker directly) nothing is tracked.
- Completed when the flow saved a choice or changed the model route; a flow that returns without
- either failed with the class it reported, else the user backed out (``abandoned``). Back (Left
- arrow) leaves the flow open: the replayed picker re-entering the same provider continues it (one
- ``started``); picking another provider, or leaving the entry point, ends it ``abandoned``.
- """
- surface = getattr(_cli, "surface", None)
- pending, _cli.backed_out = getattr(_cli, "backed_out", None), None
- if pending is not None and not _resumes(pending[0], surface, provider):
- finish_provider_setup(pending[0], "abandoned")
- pending = None
- flow, before = pending or (begin_provider_setup(surface, provider) if surface else None, None)
- if flow is None:
- yield
- return
- if pending is None:
- with contextlib.suppress(Exception):
- before = _model_route()
- hints: dict[str, Any] = {"saved": False, "failure": None}
- _cli.hints = hints
- try:
- yield
- except BaseException as exc:
- _cli.hints = None
- if _is_go_back(exc):
- _cli.backed_out = (flow, before)
- else:
- finish_provider_setup(flow, "failed", setup_failure_class(exc))
- raise
- _cli.hints = None
- landed = hints["saved"]
- if not landed:
- with contextlib.suppress(Exception):
- landed = before is not None and _model_route() != before
- if landed:
- finish_provider_setup(flow, "completed")
- elif hints["failure"]:
- finish_provider_setup(flow, "failed", hints["failure"])
- else:
- finish_provider_setup(flow, "abandoned")
- # ---- dashboard / Desktop OAuth sessions ------------------------------------------------------
- def web_setup_surface() -> str:
- from hermes_cli.process_identity import is_desktop_owned_backend
- return "desktop" if is_desktop_owned_backend() else "dashboard"
- def begin_oauth_setup(provider: str, hermes_home: Any) -> SetupFlow | None:
- try:
- return begin_provider_setup(web_setup_surface(), provider, hermes_home=hermes_home)
- except Exception:
- logger.debug("OAuth setup start not recorded", exc_info=True)
- return None
- def attach_oauth_setup(sess: dict[str, Any] | None, flow: SetupFlow | None) -> None:
- if flow is not None and isinstance(sess, dict):
- sess[_OAUTH_SESSION_KEY] = flow
- def note_oauth_failure(sess: dict[str, Any] | None, exc: BaseException) -> None:
- """A sign-in poller died with ``exc``: keep its closed class for the settle (the status alone is
- ``error`` for a lapsed code, a dropped network and a refusal alike)."""
- with contextlib.suppress(Exception):
- if isinstance(sess, dict):
- sess[_OAUTH_FAILURE_KEY] = setup_failure_class(exc)
- def settle_oauth_setup(sess: dict[str, Any] | None, *, abandoned: bool = False) -> None:
- """End an OAuth session's flow from its status (no-op while pending, or when already ended)."""
- try:
- if not isinstance(sess, dict) or sess.get(_OAUTH_SESSION_KEY) is None:
- return
- status = "cancelled" if sess.get("cancelled") else str(sess.get("status") or "")
- ending = _OAUTH_REASONS.get(str(sess.get("reason") or "")) if status in {"denied", "error"} else None
- ending = ending or _OAUTH_ENDINGS.get(status)
- if ending is None and status == "error":
- ending = ("failed", sess.get(_OAUTH_FAILURE_KEY) or "other")
- if ending is None:
- if not abandoned:
- return
- ending = ("abandoned", "none")
- flow = sess.pop(_OAUTH_SESSION_KEY, None)
- finish_provider_setup(flow, *ending)
- except Exception:
- logger.debug("OAuth setup end not recorded", exc_info=True)
|