test_achievement_engine.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221
  1. import importlib.util
  2. import unittest
  3. from pathlib import Path
  4. from tempfile import TemporaryDirectory
  5. from unittest.mock import patch
  6. MODULE_PATH = Path(__file__).resolve().parents[1] / "dashboard" / "plugin_api.py"
  7. spec = importlib.util.spec_from_file_location("plugin_api", MODULE_PATH)
  8. plugin_api = importlib.util.module_from_spec(spec)
  9. spec.loader.exec_module(plugin_api)
  10. class AchievementEngineTests(unittest.TestCase):
  11. def test_tool_call_stats_detect_tool_names_and_errors(self):
  12. messages = [
  13. {"role": "assistant", "tool_calls": [{"function": {"name": "terminal"}}]},
  14. {"role": "tool", "tool_name": "terminal", "content": "Error: port 3000 already in use"},
  15. {"role": "assistant", "tool_calls": [{"function": {"name": "web_search"}}]},
  16. ]
  17. stats = plugin_api.analyze_messages("s1", "Fix dev server", messages)
  18. self.assertEqual(stats["tool_call_count"], 2)
  19. self.assertEqual(stats["tool_names"], {"terminal", "web_search"})
  20. self.assertEqual(stats["error_count"], 1)
  21. self.assertIs(stats["port_conflict"], True)
  22. def test_tiered_achievement_reaches_highest_matching_tier(self):
  23. definition = {
  24. "id": "let_him_cook",
  25. "threshold_metric": "max_tool_calls_in_session",
  26. "tiers": [
  27. {"name": "Copper", "threshold": 10},
  28. {"name": "Silver", "threshold": 25},
  29. {"name": "Gold", "threshold": 50},
  30. ],
  31. }
  32. aggregate = {"max_tool_calls_in_session": 28}
  33. result = plugin_api.evaluate_tiered(definition, aggregate)
  34. self.assertIs(result["unlocked"], True)
  35. self.assertEqual(result["tier"], "Silver")
  36. self.assertEqual(result["progress"], 28)
  37. self.assertEqual(result["next_tier"], "Gold")
  38. def test_tiered_achievement_can_be_discovered_without_unlocking(self):
  39. definition = {
  40. "id": "terminal_goblin",
  41. "threshold_metric": "total_terminal_calls",
  42. "tiers": [{"name": "Copper", "threshold": 50}],
  43. }
  44. aggregate = {"total_terminal_calls": 12}
  45. result = plugin_api.evaluate_tiered(definition, aggregate)
  46. self.assertIs(result["unlocked"], False)
  47. self.assertIs(result["discovered"], True)
  48. self.assertEqual(result["state"], "discovered")
  49. self.assertEqual(result["progress"], 12)
  50. self.assertEqual(result["next_threshold"], 50)
  51. def test_finished_rescan_keeps_persisted_unlock_when_live_metric_shrinks(self):
  52. definition = {
  53. "id": "durable_unlock",
  54. "name": "Durable Unlock",
  55. "threshold_metric": "total_terminal_calls",
  56. "tiers": [{"name": "Copper", "threshold": 40}],
  57. }
  58. with TemporaryDirectory() as data_dir, patch.object(plugin_api, "ACHIEVEMENTS", [definition]), patch.object(plugin_api, "_data_dir", return_value=Path(data_dir)), patch.object(plugin_api, "get_hermes_home", return_value=Path(data_dir)):
  59. unlocked = plugin_api._compute_from_scan({"aggregate": {"total_terminal_calls": 40}, "sessions": []})
  60. rescanned = plugin_api._compute_from_scan({"aggregate": {"total_terminal_calls": 39}, "sessions": []})
  61. partial = plugin_api._compute_from_scan({"aggregate": {"total_terminal_calls": 39}, "sessions": []}, is_partial=True)
  62. persisted = plugin_api.load_state()["unlocks"]
  63. self.assertTrue(unlocked["achievements"][0]["unlocked"])
  64. self.assertTrue(rescanned["achievements"][0]["unlocked"])
  65. self.assertEqual(rescanned["achievements"][0]["state"], "unlocked")
  66. # In-flight snapshots are published to the cache during rescans: the floor applies there too.
  67. self.assertTrue(partial["achievements"][0]["unlocked"])
  68. self.assertEqual(partial["unlocked_count"], 1)
  69. self.assertEqual(list(persisted), ["durable_unlock"])
  70. def test_secret_achievement_stays_hidden_without_progress(self):
  71. definition = {
  72. "id": "permission_denied_any_percent",
  73. "name": "Permission Denied Any%",
  74. "secret": True,
  75. "requirements": [{"metric": "permission_denied_events", "gte": 3}],
  76. }
  77. aggregate = {"permission_denied_events": 0}
  78. result = plugin_api.evaluate_requirements(definition, aggregate)
  79. display = plugin_api.display_achievement({**definition, **result})
  80. self.assertEqual(result["state"], "secret")
  81. self.assertEqual(display["name"], "???")
  82. self.assertNotIn("Permission", display["description"])
  83. def test_multi_condition_unlock_requires_all_requirements(self):
  84. definition = {
  85. "id": "full_send",
  86. "requirements": [
  87. {"metric": "max_terminal_calls_in_session", "gte": 10},
  88. {"metric": "max_file_tool_calls_in_session", "gte": 5},
  89. {"metric": "max_web_calls_in_session", "gte": 2},
  90. ],
  91. }
  92. partial = plugin_api.evaluate_requirements(definition, {
  93. "max_terminal_calls_in_session": 12,
  94. "max_file_tool_calls_in_session": 2,
  95. "max_web_calls_in_session": 0,
  96. })
  97. complete = plugin_api.evaluate_requirements(definition, {
  98. "max_terminal_calls_in_session": 12,
  99. "max_file_tool_calls_in_session": 6,
  100. "max_web_calls_in_session": 2,
  101. })
  102. self.assertEqual(partial["state"], "discovered")
  103. self.assertIs(partial["unlocked"], False)
  104. self.assertLess(partial["progress_pct"], 100)
  105. self.assertEqual(complete["state"], "unlocked")
  106. self.assertIs(complete["unlocked"], True)
  107. def test_catalog_has_60_plus_unique_achievements(self):
  108. ids = [achievement["id"] for achievement in plugin_api.ACHIEVEMENTS]
  109. self.assertGreaterEqual(len(ids), 60)
  110. self.assertEqual(len(ids), len(set(ids)))
  111. def test_model_provider_metrics_are_aggregated(self):
  112. sessions = [
  113. {"model_names": {"openai/gpt-5", "anthropic/claude-sonnet-4"}},
  114. {"model_names": {"google/gemini-pro", "mistral/large"}},
  115. {"model_names": {"qwen/qwen3"}},
  116. ]
  117. aggregate = plugin_api.aggregate_stats(sessions)
  118. self.assertEqual(aggregate["distinct_model_count"], 5)
  119. self.assertEqual(aggregate["distinct_provider_count"], 5)
  120. result = plugin_api.evaluate_definition(
  121. next(a for a in plugin_api.ACHIEVEMENTS if a["id"] == "five_model_flight"),
  122. aggregate,
  123. )
  124. self.assertEqual(result["state"], "unlocked")
  125. self.assertEqual(result["tier"], "Copper")
  126. def test_removed_noisy_achievements_are_not_in_catalog(self):
  127. ids = {achievement["id"] for achievement in plugin_api.ACHIEVEMENTS}
  128. self.assertNotIn("fallback_pilot", ids)
  129. self.assertNotIn("browser_sleuth", ids)
  130. self.assertNotIn("release_ritualist", ids)
  131. def test_open_weights_pilgrim_counts_only_local_model_metadata(self):
  132. aggregate_mentions_only = plugin_api.aggregate_stats([
  133. {"model_names": {"openai/gpt-5"}, "local_model_events": 999},
  134. ])
  135. aggregate_local_chat = plugin_api.aggregate_stats([
  136. {"model_names": {"openai/gpt-5"}},
  137. {"model_names": {"ollama/llama3"}},
  138. ])
  139. definition = next(a for a in plugin_api.ACHIEVEMENTS if a["id"] == "open_weights_pilgrim")
  140. self.assertEqual(aggregate_mentions_only["local_model_chat_sessions"], 0)
  141. self.assertEqual(plugin_api.evaluate_definition(definition, aggregate_mentions_only)["state"], "discovered")
  142. self.assertEqual(aggregate_local_chat["local_model_chat_sessions"], 1)
  143. self.assertEqual(plugin_api.evaluate_definition(definition, aggregate_local_chat)["state"], "unlocked")
  144. def test_config_surgeon_ignores_generic_config_mentions(self):
  145. stats = plugin_api.analyze_messages("s1", "Config talk", [{"content": "config config configuration not configured"}])
  146. self.assertEqual(stats["config_events"], 0)
  147. stats = plugin_api.analyze_messages("s2", "Real config", [{"content": "edited config.yaml, manifest.json, and .env.local"}])
  148. self.assertGreaterEqual(stats["config_events"], 3)
  149. def test_dashboard_card_hover_does_not_move_click_target(self):
  150. style_css = (
  151. Path(__file__).resolve().parents[1]
  152. / "dashboard"
  153. / "dist"
  154. / "style.css"
  155. ).read_text(encoding="utf-8-sig")
  156. hover_rule = next(
  157. line for line in style_css.splitlines() if line.startswith(".ha-card:hover")
  158. )
  159. self.assertNotIn("transform:", hover_rule)
  160. self.assertIn("border-color: var(--ha-tier)", hover_rule)
  161. self.assertIn("box-shadow:", hover_rule)
  162. if __name__ == "__main__":
  163. unittest.main()
  164. class CompactionScanTests(unittest.TestCase):
  165. def test_scan_stats_survive_compaction_and_v1_checkpoint_is_rescanned(self):
  166. """#112273: compaction archives the active rows (active=0, compacted=1); the scan must
  167. keep counting them, and a schema-1 (active-only) checkpoint must not be reused."""
  168. import hermes_state
  169. from hermes_state import SessionDB
  170. with TemporaryDirectory() as tmp, patch.object(plugin_api, "_data_dir", return_value=Path(tmp) / "data"), patch.object(plugin_api, "get_hermes_home", return_value=Path(tmp)):
  171. db = SessionDB(Path(tmp) / "state.db")
  172. try:
  173. db.create_session("s1", "cli", model="m")
  174. for i in range(20):
  175. db.append_message("s1", "assistant", tool_calls=[{"function": {"name": f"tool_{i}", "arguments": "{}"}}])
  176. db.append_message("s1", "tool", content="ok", tool_name=f"tool_{i}")
  177. db.archive_and_compact("s1", [{"role": "user", "content": "[summary]"}])
  178. finally:
  179. db.close()
  180. # A schema-1 (active-only) checkpoint for this session must be ignored, not reused.
  181. stale = {"fingerprint": None, "stats": {"tool_call_count": 0}}
  182. plugin_api._write_json(plugin_api.CHECKPOINT_FILE, {"schema_version": 1, "generated_at": 1, "sessions": {"s1": stale}})
  183. with patch.object(hermes_state, "SessionDB", lambda read_only=True: SessionDB(Path(tmp) / "state.db", read_only=read_only)):
  184. scan = plugin_api.scan_sessions()
  185. self.assertEqual(scan["aggregate"]["max_distinct_tools_in_session"], 20)
  186. self.assertEqual(scan["scan_meta"]["sessions_reused"], 0)