model_setup_flows_local.py 8.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188
  1. """``hermes model`` → Local models: the desktop's Local Models setup, in the terminal.
  2. The flow drives the handlers behind the desktop's Local Models pane
  3. (``hermes_cli/web_routers/local_models.py``) in-process, so both surfaces run one code path:
  4. the catalog priced for this machine, quickstart (pinned engine via PM, model download, server
  5. start, default assignment) and activate for a model already on disk. Progress renders from the
  6. same job view the desktop polls; Ctrl+C pauses a download, and the next run resumes from the
  7. bytes already on disk.
  8. """
  9. from __future__ import annotations
  10. import asyncio
  11. import time
  12. _POLL_S = 0.2
  13. _PAUSE_WAIT_S = 15.0
  14. def local_models_available() -> bool:
  15. """Whether PM pins an engine for this machine at all. CPU is pinned on every target that has
  16. any build, so its absence means no backend can install; GPU detection waits for selection."""
  17. try:
  18. from hermes_cli.local_runtime.binaries import unavailable_reason
  19. return unavailable_reason("cpu") is None
  20. except Exception:
  21. return False
  22. def _fmt_eta(seconds: int) -> str:
  23. minutes, secs = divmod(int(seconds), 60)
  24. return f"{minutes}m{secs:02d}s" if minutes else f"{secs}s"
  25. def _rate(bytes_per_sec: float) -> str:
  26. mb = bytes_per_sec / (1 << 20)
  27. return f"{mb / 1024:.1f} GB/s" if mb >= 1024 else f"{mb:.0f} MB/s" if mb >= 10 else f"{mb:.1f} MB/s"
  28. def _progress_line(lm, view: dict) -> str:
  29. line = f" {view.get('detail') or 'Starting'}"
  30. if view.get("phase") not in lm._DOWNLOAD_PHASES:
  31. return line # bytes stay on the job after the transfer; later steps show no counter
  32. total = view.get("total_bytes")
  33. if total:
  34. line += f" {view.get('percent', 0)}% {lm._human_gb(view.get('done_bytes') or 0)} / {lm._human_gb(total)}"
  35. if view.get("bytes_per_sec"):
  36. line += f" {_rate(view['bytes_per_sec'])}"
  37. if view.get("eta_seconds") is not None:
  38. line += f" ~{_fmt_eta(view['eta_seconds'])} left"
  39. return line
  40. def _follow_job(lm, job_id: str) -> str:
  41. """Render a job until it settles; returns its terminal status. Ctrl+C pauses a pausable
  42. phase (the downloader keeps finished bytes) and stops following anything else."""
  43. shown_phase, width = None, 0
  44. try:
  45. while True:
  46. with lm._JOBS_LOCK:
  47. view = lm._job_view(lm._JOBS[job_id])
  48. if view["status"] != "running":
  49. break
  50. if view["phase"] != shown_phase:
  51. if shown_phase is not None:
  52. print()
  53. shown_phase, width = view["phase"], 0
  54. line = _progress_line(lm, view)
  55. print(f"\r{line}{' ' * max(0, width - len(line))}", end="", flush=True)
  56. width = len(line)
  57. time.sleep(_POLL_S)
  58. except KeyboardInterrupt:
  59. print()
  60. running = lm._RUNNING.get(job_id, {})
  61. with lm._JOBS_LOCK:
  62. pausable = lm._job_view(lm._JOBS[job_id])["can_pause"]
  63. if not pausable or running.get("pause") is None:
  64. print(" Stopped following. Setup continues only while this process runs.")
  65. return "interrupted"
  66. running["pause"].set()
  67. deadline = time.monotonic() + _PAUSE_WAIT_S
  68. while time.monotonic() < deadline and lm._JOBS[job_id]["status"] == "running":
  69. time.sleep(_POLL_S)
  70. print(" Paused. Run `hermes model` → Local models again to resume; finished bytes are kept.")
  71. return "paused"
  72. if shown_phase is not None:
  73. print()
  74. if view["status"] == "error":
  75. print(f" ✗ {view.get('error') or 'Setup failed'}")
  76. elif view["status"] == "done":
  77. print(f" ✓ {view.get('detail')}")
  78. return view["status"]
  79. def _start(lm, start) -> str | None:
  80. """Run a route handler that starts a job; its HTTP refusal becomes a printed line."""
  81. from fastapi import HTTPException
  82. try:
  83. result = start()
  84. if asyncio.iscoroutine(result):
  85. result = asyncio.run(result)
  86. except HTTPException as exc:
  87. print(f" ✗ {exc.detail}")
  88. return None
  89. return result.get("job_id")
  90. def _row_label(row: dict, active_id: str | None) -> str:
  91. label = f"{'★ ' if row.get('recommended') else ' '}{row['display_name']} ({row['size_label']}"
  92. label += ", downloaded)" if row.get("downloaded") else ")"
  93. label += f" — {row.get('fit_summary', '')}"
  94. if active_id and active_id in (row.get("downloaded_model_id"), row.get("model_id")):
  95. label += " ← currently active"
  96. return label
  97. def _model_flow_local(config, current_model=""):
  98. """Pick a catalog model sized for this machine (or one already on disk) and make it the default,
  99. installing the pinned engine and downloading the model as needed."""
  100. del config, current_model
  101. from hermes_cli.main_provider_setup import _prompt_provider_choice
  102. from hermes_cli.setup import prompt_yes_no
  103. from hermes_cli.web_routers import local_models as lm
  104. hw = lm.local_models_hardware()
  105. status = lm.local_models_status()
  106. gpu = f"{hw['gpu_name']} · " if hw.get("gpu_name") else ""
  107. print(f" Machine: {gpu}{hw.get('vram_label')} {'unified' if hw.get('uma') else 'GPU'} memory · "
  108. f"{lm._human_gb(hw.get('ram_total_bytes') or 0)} RAM")
  109. if status["runtime_installed"]:
  110. print(f" Engine: llama.cpp {status['tag']} ({status['runtime_backend']})"
  111. + (" (update available)" if status.get("update_available") else ""))
  112. else:
  113. print(f" Engine: not installed; llama.cpp {status['configured_tag']} installs with your first model")
  114. print()
  115. rows = lm.local_models_catalog()["models"]
  116. choices = [r for r in rows if r.get("fits") and not r.get("needs_engine")]
  117. catalog_ids = {r.get("downloaded_model_id") for r in rows}
  118. staged = [m for m in status["models"] if m["id"] not in catalog_ids]
  119. active_id = status.get("active_model_id")
  120. labels = [_row_label(r, active_id) for r in choices]
  121. labels += [f" {m['id']} — {m['size_label']} · on disk" + (" ← currently active" if m["id"] == active_id else "")
  122. for m in staged]
  123. labels.append("Leave unchanged")
  124. skipped = len(rows) - len(choices)
  125. if skipped:
  126. print(f" {skipped} catalog model(s) need more memory or a newer engine than this machine has.")
  127. default = next((i for i, r in enumerate(choices)
  128. if active_id and active_id in (r.get("downloaded_model_id"), r.get("model_id"))),
  129. next((i for i, r in enumerate(choices) if r.get("recommended")), 0))
  130. idx = _prompt_provider_choice(labels, default=default, title="Select a local model (★ recommended for this machine):")
  131. if idx is None or idx == len(labels) - 1:
  132. print("No change.")
  133. return
  134. if idx < len(choices):
  135. row = choices[idx]
  136. name, use_downloaded = row["display_name"], row.get("downloaded") and status["runtime_installed"]
  137. if not use_downloaded:
  138. need = [] if status["runtime_installed"] else [f"the llama.cpp engine ({status['configured_tag']})"]
  139. if not row.get("downloaded"):
  140. need.append(f"{name} ({row['size_label']})")
  141. if need and not prompt_yes_no(f" Download {' and '.join(need)}?", True):
  142. print("No change.")
  143. return
  144. job_id = _start(lm, lambda: lm.local_models_quickstart(lm.QuickstartBody(model_id=row["id"])))
  145. if job_id is None or _follow_job(lm, job_id) != "done":
  146. return
  147. print(f"Default model set to: {name} (via Local models · llama.cpp)")
  148. return
  149. model_id = row["downloaded_model_id"]
  150. else:
  151. model_id = staged[idx - len(choices)]["id"]
  152. name = model_id
  153. if not status["runtime_installed"]:
  154. if not prompt_yes_no(f" Download the llama.cpp engine ({status['configured_tag']})?", True):
  155. print("No change.")
  156. return
  157. job_id = _start(lm, lambda: lm.local_models_runtime_install(lm.RuntimeInstallBody()))
  158. if job_id is None or _follow_job(lm, job_id) != "done":
  159. return
  160. job_id = _start(lm, lambda: lm.local_models_activate(lm.ModelActivateBody(model_id=model_id)))
  161. if job_id is not None and _follow_job(lm, job_id) == "done":
  162. print(f"Default model set to: {name} (via Local models · llama.cpp)")