"""llms-in-the-soc-alert-triage-what-papers-measured.py - read what published papers measured for language models in security operations work, rebuild a triage confusion matrix from a paper's reported rates, and score your own triage pilot the same way. What it does: (default) prints the figures in the shipped CSV, grouped by task, one line per figure with its paper. --rebuild rebuilds the ALERT-BENCH confusion matrices (arXiv:2610.10608) for the best studied approach and for AIDA from the class counts and the reported precision / recall / F1, lists every false-positive count consistent with the rounded figures, and checks the result against the paper's own investigation load and negative predictive value. --pilot F scores a pilot of your own: a CSV with one row per alert and the columns truth attack | benign (what the alert turned out to be, decided by your analysts) decision escalate | dismiss (what the model-assisted triage did) records_returned optional integer: how many records the investigation's searches returned and prints recall, false-negative rate (with a Wilson 95% interval), precision, investigation load, negative predictive value, F1, and, when records_returned is present, the dismissal rate of attack alerts with and without any returned records (the failure pattern ALERT-BENCH found). It reads only local CSV files. It makes no network connections and calls no model. Standard library only (Python 3.9+). Inputs: --data path to the shipped CSV (default: datasets/llms-in-the-soc-alert-triage-what-papers-measured.csv) Run: python code/llms-in-the-soc-alert-triage-what-papers-measured.py python code/llms-in-the-soc-alert-triage-what-papers-measured.py --rebuild python code/llms-in-the-soc-alert-triage-what-papers-measured.py --pilot my_pilot.csv """ import argparse import csv import math import os import sys from collections import defaultdict HERE = os.path.dirname(os.path.abspath(__file__)) DEFAULT_DATA = os.path.join(HERE, "..", "datasets", "llms-in-the-soc-alert-triage-what-papers-measured.csv") # ALERT-BENCH evaluation sample, arXiv:2610.10608 v1, Section III: 225 attack-related and 1,022 benign alerts. ALERT_BENCH_POS, ALERT_BENCH_NEG = 225, 1022 def load(path): with open(path, encoding="utf-8", newline="") as f: return list(csv.DictReader(f)) def show_papers(rows): by_task = defaultdict(list) for r in rows: by_task[r["task"]].append(r) for task, rs in by_task.items(): print(f"\n== {task} ==") for r in rs: n = f" (n={r['n']})" if r["n"] else "" own = " [authors' own system]" if r["system_by_authors"] == "yes" else "" print(f" {r['paper']:<30} {r['model'][:34]:<34} {r['metric'][:58]:<58} {r['value']}{n}{own}") def metrics(tp, fn, fp, tn): pos, neg = tp + fn, fp + tn esc, dis = tp + fp, fn + tn recall = tp / pos if pos else float("nan") precision = tp / esc if esc else float("nan") f1 = 2 * precision * recall / (precision + recall) if esc and tp else 0.0 return {"recall": recall, "fnr": fn / pos if pos else float("nan"), "precision": precision, "f1": f1, "load": esc / (pos + neg), "npv": tn / dis if dis else float("nan")} def rebuild(pos, neg, recall, precision=None, f1=None): """Every (TP, FN, FP, TN) whose rates round to the reported ones (three decimals).""" tp = round(recall * pos) out = [] for fp in range(neg + 1): m = metrics(tp, pos - tp, fp, neg - fp) if round(m["recall"], 3) != round(recall, 3): continue if precision is not None and round(m["precision"], 3) != round(precision, 3): continue if f1 is not None and round(m["f1"], 3) != round(f1, 3): continue out.append((tp, pos - tp, fp, neg - fp)) return out def show_rebuild(): print(f"ALERT-BENCH sample: {ALERT_BENCH_POS} attack-related + {ALERT_BENCH_NEG} benign alerts, GPT-4.1 for every system") cases = [("Direct-Prompt (best studied approach)", dict(recall=0.596, f1=0.744)), ("AIDA (the authors' design)", dict(recall=0.969, precision=0.948))] for name, rep in cases: found = rebuild(ALERT_BENCH_POS, ALERT_BENCH_NEG, **rep) print(f"\n{name}: reported {rep}") for tp, fn, fp, tn in found: m = metrics(tp, fn, fp, tn) print(f" TP {tp:>4} FN {fn:>4} FP {fp:>4} TN {tn:>5} | sent to analysts {tp + fp:>4} " f"({m['load']:.1%}) missed attacks {fn} FNR {m['fnr']:.1%} NPV {m['npv']:.1%} F1 {m['f1']:.3f}") print("\nPaper's own figures to compare: AIDA load 18.4%, NPV 99.3%, FNR 3.1%; studied approaches load 8.5-14.0%.") def wilson(k, n, z=1.96): if n == 0: return (float("nan"), float("nan")) p = k / n d = 1 + z * z / n c = (p + z * z / (2 * n)) / d h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d return (max(0.0, c - h), min(1.0, c + h)) def score_pilot(path): rows = load(path) tp = fn = fp = tn = 0 empty = {"dismissed": 0, "total": 0} some = {"dismissed": 0, "total": 0} has_records = rows and "records_returned" in rows[0] for i, r in enumerate(rows, 2): t, d = r["truth"].strip().lower(), r["decision"].strip().lower() if t not in ("attack", "benign") or d not in ("escalate", "dismiss"): sys.exit(f"line {i}: truth must be attack|benign and decision escalate|dismiss, got {t!r}, {d!r}") if t == "attack": if d == "escalate": tp += 1 else: fn += 1 if has_records and r["records_returned"].strip() != "": bucket = empty if int(r["records_returned"]) == 0 else some bucket["total"] += 1 bucket["dismissed"] += d == "dismiss" else: if d == "escalate": fp += 1 else: tn += 1 m = metrics(tp, fn, fp, tn) lo, hi = wilson(fn, tp + fn) print(f"alerts {len(rows)}: {tp + fn} attack, {fp + tn} benign") print(f"recall {m['recall']:.3f} false-negative rate {m['fnr']:.1%} (95% interval {lo:.1%} to {hi:.1%})") print(f"precision {m['precision']:.3f} F1 {m['f1']:.3f}") print(f"investigation load {m['load']:.1%} dismissed alerts that were benign (NPV) {m['npv']:.1%}") if has_records: for label, b in (("no records returned", empty), ("some records returned", some)): if b["total"]: print(f"attack alerts with {label}: {b['dismissed']}/{b['total']} dismissed ({b['dismissed'] / b['total']:.1%})") if tp + fn < 30: print("note: fewer than 30 attack alerts; the false-negative rate is too uncertain to act on alone.") def main(): ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) ap.add_argument("--data", default=DEFAULT_DATA) ap.add_argument("--rebuild", action="store_true") ap.add_argument("--pilot") a = ap.parse_args() if a.pilot: score_pilot(a.pilot) elif a.rebuild: show_rebuild() else: show_papers(load(a.data)) if __name__ == "__main__": main()