evidence-store.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313
  1. #!/usr/bin/env python3
  2. """
  3. OSS Forensics Evidence Store Manager
  4. Manages a JSON-based evidence store for forensic investigations.
  5. Commands:
  6. add - Add a piece of evidence
  7. list - List all evidence (optionally filter by type or actor)
  8. verify - Re-check SHA-256 hashes for integrity
  9. query - Search evidence by keyword
  10. export - Export evidence as a Markdown table
  11. summary - Print investigation statistics
  12. Usage example:
  13. python3 evidence-store.py --store evidence.json add \
  14. --source "git fsck output" --content "dangling commit abc123" \
  15. --type git --actor "malicious-user" --url "https://github.com/owner/repo/commit/abc123"
  16. python3 evidence-store.py --store evidence.json list --type git
  17. python3 evidence-store.py --store evidence.json verify
  18. python3 evidence-store.py --store evidence.json export > evidence-table.md
  19. """
  20. import json
  21. import argparse
  22. import os
  23. import datetime
  24. import hashlib
  25. import sys
  26. EVIDENCE_TYPES = [
  27. "git", # Local git repository data (commits, reflog, fsck)
  28. "gh_api", # GitHub REST API responses
  29. "gh_archive", # GitHub Archive / BigQuery query results
  30. "web_archive", # Wayback Machine snapshots
  31. "ioc", # Indicator of Compromise (SHA, domain, IP, package name, etc.)
  32. "analysis", # Derived analysis / cross-source correlation result
  33. "manual", # Manually noted observation
  34. "vendor_report", # External security vendor report excerpt
  35. ]
  36. VERIFICATION_STATES = ["unverified", "single_source", "multi_source_verified"]
  37. IOC_TYPES = [
  38. "COMMIT_SHA", "FILE_PATH", "API_KEY", "SECRET", "IP_ADDRESS",
  39. "DOMAIN", "PACKAGE_NAME", "ACTOR_USERNAME", "MALICIOUS_URL",
  40. "WORKFLOW_FILE", "BRANCH_NAME", "TAG_NAME", "RELEASE_NAME", "OTHER",
  41. ]
  42. def _now_iso():
  43. return datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds") + "Z"
  44. def _sha256(content: str) -> str:
  45. return hashlib.sha256(content.encode("utf-8")).hexdigest()
  46. class EvidenceStore:
  47. def __init__(self, filepath: str):
  48. self.filepath = filepath
  49. self.data = {
  50. "metadata": {
  51. "version": "2.0",
  52. "created_at": _now_iso(),
  53. "last_updated": _now_iso(),
  54. "investigation": "",
  55. "target_repo": "",
  56. },
  57. "evidence": [],
  58. "chain_of_custody": [],
  59. }
  60. if os.path.exists(filepath):
  61. try:
  62. with open(filepath, "r", encoding="utf-8") as f:
  63. self.data = json.load(f)
  64. except (OSError, json.JSONDecodeError) as e:
  65. print(f"Error loading evidence store '{filepath}': {e}", file=sys.stderr)
  66. print("Hint: The file might be corrupted. Check for manual edits or syntax errors.", file=sys.stderr)
  67. sys.exit(1)
  68. def _save(self):
  69. self.data["metadata"]["last_updated"] = _now_iso()
  70. with open(self.filepath, "w", encoding="utf-8") as f:
  71. json.dump(self.data, f, indent=2, ensure_ascii=False)
  72. def _next_id(self) -> str:
  73. return f"EV-{len(self.data['evidence']) + 1:04d}"
  74. def add(
  75. self,
  76. source: str,
  77. content: str,
  78. evidence_type: str,
  79. actor: str | None = None,
  80. url: str | None = None,
  81. timestamp: str | None = None,
  82. ioc_type: str | None = None,
  83. verification: str = "unverified",
  84. notes: str | None = None,
  85. ) -> str:
  86. evidence_id = self._next_id()
  87. entry = {
  88. "id": evidence_id,
  89. "type": evidence_type,
  90. "source": source,
  91. "content": content,
  92. "content_sha256": _sha256(content),
  93. "actor": actor,
  94. "url": url,
  95. "event_timestamp": timestamp,
  96. "collected_at": _now_iso(),
  97. "ioc_type": ioc_type,
  98. "verification": verification,
  99. "notes": notes,
  100. }
  101. self.data["evidence"].append(entry)
  102. self.data["chain_of_custody"].append({
  103. "action": "add",
  104. "evidence_id": evidence_id,
  105. "timestamp": _now_iso(),
  106. "source": source,
  107. })
  108. self._save()
  109. return evidence_id
  110. def list_evidence(self, filter_type: str | None = None, filter_actor: str | None = None):
  111. results = self.data["evidence"]
  112. if filter_type:
  113. results = [e for e in results if e.get("type") == filter_type]
  114. if filter_actor:
  115. results = [e for e in results if e.get("actor") == filter_actor]
  116. return results
  117. def verify_integrity(self):
  118. """Re-compute SHA-256 for all entries and report mismatches."""
  119. issues = []
  120. for entry in self.data["evidence"]:
  121. expected = _sha256(entry["content"])
  122. stored = entry.get("content_sha256", "")
  123. if expected != stored:
  124. issues.append({
  125. "id": entry["id"],
  126. "stored_sha256": stored,
  127. "computed_sha256": expected,
  128. })
  129. return issues
  130. def query(self, keyword: str):
  131. """Search for keyword in content, source, actor, or url."""
  132. keyword_lower = keyword.lower()
  133. return [
  134. e for e in self.data["evidence"]
  135. if keyword_lower in (e.get("content", "") or "").lower()
  136. or keyword_lower in (e.get("source", "") or "").lower()
  137. or keyword_lower in (e.get("actor", "") or "").lower()
  138. or keyword_lower in (e.get("url", "") or "").lower()
  139. ]
  140. def export_markdown(self) -> str:
  141. lines = [
  142. "# Evidence Registry",
  143. "",
  144. f"**Store**: `{self.filepath}`",
  145. f"**Last Updated**: {self.data['metadata'].get('last_updated', 'N/A')}",
  146. f"**Total Evidence Items**: {len(self.data['evidence'])}",
  147. "",
  148. "| ID | Type | Source | Actor | Verification | Event Timestamp | URL |",
  149. "|----|------|--------|-------|--------------|-----------------|-----|",
  150. ]
  151. for e in self.data["evidence"]:
  152. url = e.get("url") or ""
  153. url_display = f"[link]({url})" if url else ""
  154. lines.append(
  155. f"| {e['id']} | {e.get('type','')} | {e.get('source','')} "
  156. f"| {e.get('actor') or ''} | {e.get('verification','')} "
  157. f"| {e.get('event_timestamp') or ''} | {url_display} |"
  158. )
  159. lines.append("")
  160. lines.append("## Chain of Custody")
  161. lines.append("")
  162. lines.append("| Evidence ID | Action | Timestamp | Source |")
  163. lines.append("|-------------|--------|-----------|--------|")
  164. for c in self.data["chain_of_custody"]:
  165. lines.append(
  166. f"| {c.get('evidence_id','')} | {c.get('action','')} "
  167. f"| {c.get('timestamp','')} | {c.get('source','')} |"
  168. )
  169. return "\n".join(lines)
  170. def summary(self) -> dict:
  171. by_type = {}
  172. by_verification = {}
  173. actors = set()
  174. for e in self.data["evidence"]:
  175. t = e.get("type", "unknown")
  176. by_type[t] = by_type.get(t, 0) + 1
  177. v = e.get("verification", "unverified")
  178. by_verification[v] = by_verification.get(v, 0) + 1
  179. if e.get("actor"):
  180. actors.add(e["actor"])
  181. return {
  182. "total": len(self.data["evidence"]),
  183. "by_type": by_type,
  184. "by_verification": by_verification,
  185. "unique_actors": sorted(actors),
  186. }
  187. def main():
  188. parser = argparse.ArgumentParser(
  189. description="OSS Forensics Evidence Store Manager v2.0",
  190. formatter_class=argparse.RawDescriptionHelpFormatter,
  191. )
  192. parser.add_argument("--store", default="evidence.json", help="Path to evidence JSON file (default: evidence.json)")
  193. subparsers = parser.add_subparsers(dest="command", metavar="COMMAND")
  194. # --- add ---
  195. add_p = subparsers.add_parser("add", help="Add a new evidence entry")
  196. add_p.add_argument("--source", required=True, help="Where this evidence came from (e.g. 'git fsck', 'GH API /commits')")
  197. add_p.add_argument("--content", required=True, help="The evidence content (commit SHA, API response excerpt, etc.)")
  198. add_p.add_argument("--type", required=True, choices=EVIDENCE_TYPES, dest="evidence_type", help="Evidence type")
  199. add_p.add_argument("--actor", help="GitHub handle or email of associated actor")
  200. add_p.add_argument("--url", help="URL to original source")
  201. add_p.add_argument("--timestamp", help="When the event occurred (ISO 8601)")
  202. add_p.add_argument("--ioc-type", choices=IOC_TYPES, help="IOC subtype (for --type ioc)")
  203. add_p.add_argument("--verification", choices=VERIFICATION_STATES, default="unverified")
  204. add_p.add_argument("--notes", help="Additional investigator notes")
  205. add_p.add_argument("--quiet", action="store_true", help="Suppress success message")
  206. # --- list ---
  207. list_p = subparsers.add_parser("list", help="List all evidence entries")
  208. list_p.add_argument("--type", dest="filter_type", choices=EVIDENCE_TYPES, help="Filter by type")
  209. list_p.add_argument("--actor", dest="filter_actor", help="Filter by actor")
  210. # --- verify ---
  211. subparsers.add_parser("verify", help="Verify SHA-256 integrity of all evidence content")
  212. # --- query ---
  213. query_p = subparsers.add_parser("query", help="Search evidence by keyword")
  214. query_p.add_argument("keyword", help="Keyword to search for")
  215. # --- export ---
  216. subparsers.add_parser("export", help="Export evidence as a Markdown table (stdout)")
  217. # --- summary ---
  218. subparsers.add_parser("summary", help="Print investigation statistics")
  219. args = parser.parse_args()
  220. if not args.command:
  221. parser.print_help()
  222. sys.exit(0)
  223. store = EvidenceStore(args.store)
  224. if args.command == "add":
  225. eid = store.add(
  226. source=args.source,
  227. content=args.content,
  228. evidence_type=args.evidence_type,
  229. actor=args.actor,
  230. url=args.url,
  231. timestamp=args.timestamp,
  232. ioc_type=args.ioc_type,
  233. verification=args.verification,
  234. notes=args.notes,
  235. )
  236. if not getattr(args, "quiet", False):
  237. print(f"✓ Added evidence: {eid}")
  238. elif args.command == "list":
  239. items = store.list_evidence(
  240. filter_type=getattr(args, "filter_type", None),
  241. filter_actor=getattr(args, "filter_actor", None),
  242. )
  243. if not items:
  244. print("No evidence found.")
  245. for e in items:
  246. actor_str = f" | actor: {e['actor']}" if e.get("actor") else ""
  247. url_str = f" | {e['url']}" if e.get("url") else ""
  248. print(f"[{e['id']}] {e['type']:12s} | {e['verification']:20s} | {e['source']}{actor_str}{url_str}")
  249. elif args.command == "verify":
  250. issues = store.verify_integrity()
  251. if not issues:
  252. print(f"✓ All {len(store.data['evidence'])} evidence entries passed SHA-256 integrity check.")
  253. else:
  254. print(f"✗ {len(issues)} integrity issue(s) detected:")
  255. for i in issues:
  256. print(f" [{i['id']}] stored={i['stored_sha256'][:16]}... computed={i['computed_sha256'][:16]}...")
  257. sys.exit(1)
  258. elif args.command == "query":
  259. results = store.query(args.keyword)
  260. print(f"Found {len(results)} result(s) for '{args.keyword}':")
  261. for e in results:
  262. print(f" [{e['id']}] {e['type']} | {e['source']} | {e['content'][:80]}")
  263. elif args.command == "export":
  264. print(store.export_markdown())
  265. elif args.command == "summary":
  266. s = store.summary()
  267. print(f"Total evidence items : {s['total']}")
  268. print(f"By type : {json.dumps(s['by_type'], indent=2)}")
  269. print(f"By verification : {json.dumps(s['by_verification'], indent=2)}")
  270. print(f"Unique actors : {s['unique_actors']}")
  271. if __name__ == "__main__":
  272. main()