stuck_loop_counts.py 2.1 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556
  1. """Persisted per-session restart-failure counters (``.restart_failure_counts``, #7536).
  2. Every load/mutate/save of the file runs under one lock: session completions clear their
  3. counter from worker threads concurrently, and an unserialized pair rewrites a stale snapshot
  4. that resurrects the other session's counter.
  5. """
  6. import json
  7. import logging
  8. import threading
  9. from pathlib import Path
  10. logger = logging.getLogger(__name__)
  11. # Nothing re-acquires it while held, so a plain Lock suffices.
  12. _COUNTS_LOCK = threading.Lock()
  13. def _read_counts(path: Path) -> dict:
  14. """Parsed counters, or {} when the file is missing, unreadable or not a JSON object."""
  15. try:
  16. counts = json.loads(path.read_text(encoding="utf-8-sig"))
  17. except (OSError, ValueError):
  18. return {}
  19. return counts if isinstance(counts, dict) else {}
  20. def increment_restart_failure_counts(path: Path, active_session_keys: set) -> None:
  21. """Increment the counters of sessions active at shutdown; drop the rest (their loop is broken)."""
  22. from utils import atomic_json_write
  23. with _COUNTS_LOCK:
  24. counts = _read_counts(path)
  25. try:
  26. atomic_json_write(path, {key: counts.get(key, 0) + 1 for key in active_session_keys}, indent=None)
  27. except Exception:
  28. logger.debug("Could not write restart-failure counters to %s", path, exc_info=True)
  29. def clear_restart_failure_count(path: Path, session_key: str) -> None:
  30. """Drop a completed session's counter; removes the file once no counter is left.
  31. Blocking (``atomic_json_write`` fsyncs): callers on the event loop run it via ``asyncio.to_thread``.
  32. """
  33. from utils import atomic_json_write
  34. with _COUNTS_LOCK:
  35. counts = _read_counts(path)
  36. if session_key not in counts:
  37. return
  38. del counts[session_key]
  39. try:
  40. if counts:
  41. atomic_json_write(path, counts, indent=None)
  42. else:
  43. path.unlink(missing_ok=True)
  44. except Exception:
  45. logger.debug("Could not clear restart-failure counter in %s", path, exc_info=True)