shared_metrics_catalog.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325
  1. """Public-name catalogs for shared metrics: the only non-enum names a package may carry.
  2. Every set here is something Nous itself publishes (slash-command registry, bundled and optional
  3. skills, optional-mcps/ and plugin-catalog/ entries, built-in auxiliary tasks, shipped locales).
  4. A name outside its set is reported as ``custom`` so user-defined identities never leave the machine.
  5. Loaders are cached: the catalogs only change with the installed Hermes version.
  6. """
  7. from __future__ import annotations
  8. import functools
  9. import logging
  10. import re
  11. from pathlib import Path
  12. logger = logging.getLogger(__name__)
  13. _REPO_ROOT = Path(__file__).resolve().parents[2]
  14. CUSTOM = "custom"
  15. def _skill_dir_names(root: Path) -> frozenset[str]:
  16. try:
  17. return frozenset(p.parent.name.lower() for p in root.rglob("SKILL.md"))
  18. except OSError:
  19. return frozenset()
  20. def _yaml_stems(root: Path) -> frozenset[str]:
  21. try:
  22. return frozenset(p.stem.lower() for p in root.glob("*.yaml"))
  23. except OSError:
  24. return frozenset()
  25. @functools.cache
  26. def slash_command_names() -> frozenset[str]:
  27. from hermes_cli.commands import COMMAND_REGISTRY
  28. return frozenset(command.name for command in COMMAND_REGISTRY)
  29. @functools.cache
  30. def bundled_skill_names() -> frozenset[str]:
  31. from hermes_constants import get_bundled_skills_dir, get_optional_skills_dir
  32. return _skill_dir_names(get_bundled_skills_dir(_REPO_ROOT / "skills")) | _skill_dir_names(
  33. get_optional_skills_dir(_REPO_ROOT / "optional-skills")
  34. )
  35. @functools.cache
  36. def mcp_catalog_names() -> frozenset[str]:
  37. from hermes_constants import get_optional_mcps_dir
  38. root = get_optional_mcps_dir(_REPO_ROOT / "optional-mcps")
  39. try:
  40. return frozenset(p.name.lower() for p in root.iterdir() if p.is_dir())
  41. except OSError:
  42. return frozenset()
  43. @functools.cache
  44. def plugin_catalog_names() -> frozenset[str]:
  45. return _yaml_stems(_REPO_ROOT / "plugin-catalog")
  46. @functools.cache
  47. def aux_task_names() -> frozenset[str]:
  48. from hermes_cli.main_provider_setup import _AUX_TASKS
  49. return frozenset(key for key, _name, _desc in _AUX_TASKS)
  50. @functools.cache
  51. def display_languages() -> frozenset[str]:
  52. return _yaml_stems(_REPO_ROOT / "locales")
  53. @functools.cache
  54. def provider_names() -> frozenset[str]:
  55. """Provider ids Hermes itself ships: built-in auth rows, overlays, alias tables and the
  56. in-tree ``plugins/model-providers`` profiles. Never the live registries (``PROVIDER_REGISTRY``,
  57. picker labels): ``$HERMES_HOME`` and pip provider plugins add their user-chosen names there."""
  58. import providers
  59. from hermes_cli.auth import _PROVIDER_ALIASES, BUILTIN_PROVIDER_IDS
  60. from hermes_cli.models_catalog_static import _PROVIDER_ALIASES as _CATALOG_ALIASES
  61. from hermes_cli.providers import ALIASES, HERMES_OVERLAYS
  62. providers.list_providers() # runs discovery; bundled profiles land in the process-wide layer
  63. bundled = {
  64. alias for name, profile in providers._REGISTRY.items() if providers._SOURCES.get(name) == "bundled"
  65. for alias in (name, *profile.aliases)
  66. }
  67. return BUILTIN_PROVIDER_IDS.union(
  68. HERMES_OVERLAYS, ALIASES, _PROVIDER_ALIASES, _CATALOG_ALIASES, bundled, ("openrouter", CUSTOM)
  69. )
  70. @functools.cache
  71. def user_named_model_providers() -> frozenset[str]:
  72. """Providers whose model ids the user names: custom endpoints and loopback servers."""
  73. from urllib.parse import urlparse
  74. from hermes_cli import auth, models, providers
  75. from hermes_cli.auth import PROVIDER_REGISTRY
  76. hosts = {"localhost", "127.0.0.1", "::1", "0.0.0.0"}
  77. loopback = {
  78. name for name, config in PROVIDER_REGISTRY.items()
  79. if urlparse(str(getattr(config, "inference_base_url", "") or "")).hostname in hosts
  80. }
  81. # Alias spellings (``lm-studio``) pass provider_metric_name as shipped names, so they must collapse too.
  82. tables = (providers.ALIASES, getattr(auth, "_PROVIDER_ALIASES", {}), getattr(models, "_PROVIDER_ALIASES", {}))
  83. aliases = {alias for table in tables for alias, canon in table.items() if canon in loopback}
  84. return frozenset(loopback | aliases | {CUSTOM})
  85. @functools.cache
  86. def custom_provider_aliases() -> frozenset[str]:
  87. """Provider ids Hermes routes through the generic ``custom`` provider (``ollama``, ``vllm``,
  88. ``llamacpp``...): shipped names, but the server and its model ids are the user's own."""
  89. from hermes_cli import auth, models, providers
  90. tables = (providers.ALIASES, getattr(auth, "_PROVIDER_ALIASES", {}), getattr(models, "_PROVIDER_ALIASES", {}))
  91. return frozenset(alias for table in tables for alias, canon in table.items() if canon == CUSTOM)
  92. # ---- v4 gateway ----
  93. @functools.cache
  94. def bundled_platform_names() -> frozenset[str]:
  95. """Messaging platforms Hermes ships as ``plugins/platforms/<name>`` (the dir is the registered name)."""
  96. root = _REPO_ROOT / "plugins" / "platforms"
  97. try:
  98. return frozenset(p.name.lower() for p in root.iterdir() if (p / "plugin.yaml").is_file())
  99. except OSError:
  100. return frozenset()
  101. @functools.cache
  102. def catalog_platform_names() -> frozenset[str]:
  103. """``plugin-catalog/`` entries whose category is ``platform`` (in-tree only: never a network fetch)."""
  104. from hermes_cli.plugin_catalog import CATALOG_TIERS, load_catalog
  105. return frozenset(e.name for e in load_catalog() if e.category == "platform" and e.tier in CATALOG_TIERS)
  106. def platform_metric_name(raw: object, core: frozenset[str]) -> str:
  107. """A messaging platform's public name: a core or bundled platform's own name, the catalog entry
  108. name of the installed catalog plugin that registered it, else ``plugin``. ``core`` is the
  109. contract's static platform vocabulary."""
  110. name = _norm(getattr(raw, "value", raw))
  111. if name in core or name in _safe(bundled_platform_names):
  112. return name
  113. if not name:
  114. return "plugin"
  115. from hermes_constants import get_hermes_home
  116. return _catalog_platform_owner(str(get_hermes_home()), name) or "plugin"
  117. @functools.lru_cache(maxsize=64)
  118. def _catalog_platform_owner(home: str, platform: str) -> str | None:
  119. """The catalog entry that installed the plugin registering ``platform`` in this profile. The
  120. plugin is located from where its adapter factory's code lives (not a name it could claim), and
  121. the catalog install is proven only by the installer-owned ``.install-metadata.json`` record,
  122. never by anything inside the plugin tree, so a URL install cannot claim a catalog name."""
  123. try:
  124. import inspect
  125. from gateway.platform_registry import platform_registry
  126. from hermes_cli.plugins_provenance import read_sidecar_rows
  127. entry = platform_registry.get(platform)
  128. if entry is None or getattr(entry, "source", "") != "plugin":
  129. return None
  130. plugins_dir = (Path(home) / "plugins").resolve()
  131. code_path = Path(inspect.getfile(entry.adapter_factory)).resolve()
  132. if not code_path.is_relative_to(plugins_dir) or code_path.parent == plugins_dir:
  133. return None
  134. row = read_sidecar_rows(plugins_dir).get(code_path.relative_to(plugins_dir).parts[0])
  135. row = row if isinstance(row, dict) else {}
  136. block = row.get("catalog")
  137. name = _norm(block.get("name") if isinstance(block, dict) else row.get("catalog_name"))
  138. return name if name in _safe(catalog_platform_names) else None
  139. except Exception:
  140. logger.debug("Shared-metrics platform provenance unavailable", exc_info=True)
  141. return None
  142. def _norm(value: object) -> str:
  143. return value.strip().lower() if isinstance(value, str) else ""
  144. def _safe(loader) -> frozenset[str]:
  145. try:
  146. return loader()
  147. except Exception:
  148. logger.debug("Shared-metrics catalog %s unavailable", loader.__name__, exc_info=True)
  149. return frozenset()
  150. def slash_command_metric_name(raw: object) -> str:
  151. """Canonical registry name for a command or alias; skill and plugin commands stay anonymous."""
  152. name = _norm(raw).lstrip("/").split(" ", 1)[0]
  153. if not name:
  154. return "unknown"
  155. try:
  156. from hermes_cli.commands import resolve_command
  157. command = resolve_command(name)
  158. except Exception:
  159. command = None
  160. if command is not None and command.name in _safe(slash_command_names):
  161. return command.name
  162. try:
  163. from agent.skill_commands import resolve_skill_command_key
  164. if resolve_skill_command_key(name):
  165. return "skill"
  166. from hermes_cli.plugins import get_plugin_command_handler
  167. if get_plugin_command_handler(name) is not None:
  168. return "plugin"
  169. except Exception:
  170. pass
  171. return "unknown"
  172. def skill_metric_name(raw: object) -> str:
  173. name = _norm(raw)
  174. return name if name in _safe(bundled_skill_names) else CUSTOM
  175. _EXTENSION_CATALOGS = {
  176. "skill": bundled_skill_names,
  177. "mcp_server": mcp_catalog_names,
  178. "plugin": plugin_catalog_names,
  179. }
  180. def extension_metric_name(kind: str, raw: object) -> str:
  181. """A catalog entry's public name, else ``custom`` (URL/local installs of anything)."""
  182. loader = _EXTENSION_CATALOGS.get(kind)
  183. name = _norm(raw).rsplit("/", 1)[-1]
  184. return name if loader is not None and name in _safe(loader) else CUSTOM
  185. def aux_task_metric_name(raw: object) -> str:
  186. name = _norm(raw)
  187. if not name:
  188. return "none"
  189. return name if name in _safe(aux_task_names) else "other"
  190. def provider_metric_name(raw: object) -> str:
  191. """A shipped provider id; user-named providers (``custom:<name>``, unknown ids) and the local
  192. server aliases of ``custom`` read ``custom``."""
  193. from .shared_metrics_contract import PROVIDER_IDENTIFIER_MAX_LENGTH, _metric_identifier
  194. name = _metric_identifier(raw, max_length=PROVIDER_IDENTIFIER_MAX_LENGTH)
  195. if name == "unknown":
  196. return name
  197. if name.startswith(CUSTOM) or name in _safe(custom_provider_aliases):
  198. return CUSTOM
  199. return name if name in _safe(provider_names) or _models_dev_provider(name) else CUSTOM
  200. @functools.lru_cache(maxsize=256)
  201. def _models_dev_provider(name: str) -> bool:
  202. """A public models.dev provider id, from the local cache only (never a network call)."""
  203. try:
  204. from agent.models_dev import get_provider_info
  205. return get_provider_info(name, allow_network=False) is not None
  206. except Exception:
  207. return False
  208. def model_metric_name(raw: object, provider: str, *, max_length: int) -> str:
  209. """The model id for a shipped remote provider; ``custom`` when the user names it (custom
  210. endpoint, loopback server), the provider is unknown (nothing proves the id is public), or it
  211. looks like a filesystem path or URL."""
  212. from .shared_metrics_contract import _metric_identifier
  213. if provider in _safe(user_named_model_providers):
  214. return CUSTOM
  215. model = _metric_identifier(raw, max_length=max_length)
  216. if model == "unknown":
  217. return model
  218. if provider == "unknown" or _LOCATION_MODEL.match(model):
  219. return CUSTOM
  220. # Azure calls a deployment by the name its owner chose (``acme-legal-prod``): only a public model id passes.
  221. if provider.startswith("azure") and model not in _safe(public_model_ids):
  222. return CUSTOM
  223. return model
  224. @functools.cache
  225. def public_model_ids() -> frozenset[str]:
  226. """Model ids Hermes ships in its static catalogs plus every id in the local models.dev cache
  227. (never a network call), with and without a ``vendor/`` prefix."""
  228. from agent.models_dev import fetch_models_dev
  229. from hermes_cli.models_catalog_static import _PROVIDER_MODELS
  230. ids = {model for models in _PROVIDER_MODELS.values() for model in models}
  231. for entry in fetch_models_dev(allow_network=False).values():
  232. if isinstance(entry, dict) and isinstance(entry.get("models"), dict):
  233. ids.update(entry["models"])
  234. return frozenset(form.lower() for model in ids if isinstance(model, str)
  235. for form in (model, model.rsplit("/", 1)[-1]))
  236. # The user's own server or account, never a public model id: a URL or path (``:/``), a weight file,
  237. # a loopback or IPv4 host, ``host:port`` (a 2-5 digit port, so Bedrock's ``...-v1:0`` stays readable),
  238. # or an AWS ARN (it carries the account id).
  239. _LOCATION_MODEL = re.compile(
  240. r"arn:|.*:/|.*\.(?:gguf|bin|safetensors)$|(?:localhost|\d{1,3}(?:\.\d{1,3}){3})(?:[:/]|$)|[^/:]+:\d{2,5}(?:/|$)"
  241. )
  242. def display_language_metric_name(raw: object) -> str:
  243. name = _norm(raw) or "en"
  244. return name if name in _safe(display_languages) else "other"