context_default_record.py 3.2 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071
  1. """Which context-window resolutions ended on a static default, recorded by the resolver itself.
  2. ``get_model_context_length`` has many sources (config, endpoint metadata, /models, provider catalogs,
  3. local probes, Bedrock's table and probe). Only the resolver knows whether its answer came from one of
  4. them or from a default (the generic 256K, Bedrock's 128K), so the "fallback" label reads this record
  5. instead of re-deriving the answer. Keyed by profile home: a multiplexed gateway resolves for several
  6. profiles in one process.
  7. """
  8. from __future__ import annotations
  9. import contextvars
  10. import functools
  11. import inspect
  12. from typing import Callable, Optional
  13. # Set while a recorded resolution runs; the default branches flip it via mark_context_defaulted().
  14. _DEFAULTED: contextvars.ContextVar[Optional[list]] = contextvars.ContextVar("context_defaulted", default=None)
  15. _RECORDS: dict[tuple[str, str, str], int] = {} # (profile home, model, base_url) -> the defaulted window
  16. # The calling context's last resolution: its defaulted window, or None when a source knew it. A compressor reads this
  17. # right after its own lookup and keeps it, so a later lookup on the same route cannot relabel its window.
  18. _LAST_DEFAULTED: contextvars.ContextVar[Optional[int]] = contextvars.ContextVar("last_context_defaulted", default=None)
  19. _MAX_RECORDS = 512
  20. def _key(model, base_url) -> tuple[str, str, str]:
  21. from hermes_constants import hermes_home_key
  22. return hermes_home_key(), str(model or "").strip(), str(base_url or "").strip().rstrip("/")
  23. def mark_context_defaulted() -> None:
  24. """Called by a resolver branch that returns a static default instead of a known window."""
  25. flag = _DEFAULTED.get()
  26. if flag is not None:
  27. flag[0] = True
  28. def records_context_defaults(resolve: Callable[..., int]) -> Callable[..., int]:
  29. """Wrap the resolver: remember a defaulted answer for its (model, base_url), forget it otherwise."""
  30. signature = inspect.signature(resolve)
  31. @functools.wraps(resolve)
  32. def wrapper(*args, **kwargs):
  33. outer, flag = _DEFAULTED.get(), [False]
  34. token = _DEFAULTED.set(flag)
  35. try:
  36. context_length = resolve(*args, **kwargs)
  37. finally:
  38. _DEFAULTED.reset(token)
  39. if flag[0] and outer is not None:
  40. outer[0] = True # a nested lookup (a MoA preset's aggregator) defaulted the answer the outer call returns
  41. _LAST_DEFAULTED.set(context_length if flag[0] else None)
  42. call = signature.bind_partial(*args, **kwargs).arguments
  43. key = _key(call.get("model"), call.get("base_url"))
  44. if flag[0]:
  45. if len(_RECORDS) >= _MAX_RECORDS:
  46. _RECORDS.clear()
  47. _RECORDS[key] = context_length
  48. else:
  49. _RECORDS.pop(key, None)
  50. return context_length
  51. return wrapper
  52. def last_defaulted_context_length() -> Optional[int]:
  53. """The window this context's most recent resolution defaulted to, or None when a source knew it."""
  54. return _LAST_DEFAULTED.get()
  55. def was_context_defaulted(context_length, model, base_url="") -> bool:
  56. """True when the last resolution of ``model`` on ``base_url`` in this profile defaulted to ``context_length``."""
  57. return _RECORDS.get(_key(model, base_url)) == context_length