"""stealer-logs-in-the-breach-record.py - what the public breach catalogue says about
stealer logs, and how much of recent breach volume they actually are.

Reads the Have I Been Pwned breaches endpoint (public, no key; it returns metadata
about breaches and contains no credentials), then:

  1. lists every entry flagged IsStealerLog, with its dates, unique-address count,
     data classes, and the row count its own description states, where it states one;
  2. measures the stealer-log share of loaded addresses, and of password-bearing
     addresses, by the year HIBP loaded the data (AddedDate) and by the year the
     data is dated (BreachDate), because the two answer different questions;
  3. names the largest entry of each recent year, flagged or not, so a share can be
     read against what it is being compared with.

Standard library only (urllib, json, re, csv, argparse). Python 3.9 or newer.

    python stealer-logs-in-the-breach-record.py
    python stealer-logs-in-the-breach-record.py --csv stealer-logs-in-the-breach-record.csv
"""
from __future__ import annotations

import argparse
import csv
import html
import json
import re
import urllib.request

URL = "https://haveibeenpwned.com/api/v3/breaches"
YEARS = ("2024", "2025", "2026")

# Row counts are stated in words in some descriptions ("23 billion rows", "2.7B rows").
SCALE = {"billion": 1e9, "b": 1e9, "million": 1e6, "m": 1e6}
ROWS_RE = re.compile(r"([\d.]+)\s*(billion|million|B|M)\s+rows", re.I)


def fetch():
    request = urllib.request.Request(URL, headers={"User-Agent": "passwordpassword-research/1.0"})
    with urllib.request.urlopen(request, timeout=90) as response:
        return json.loads(response.read().decode("utf-8"))


def plain(description):
    return re.sub(r"\s+", " ", html.unescape(re.sub(r"<[^>]+>", " ", description))).strip()


def stated_rows(description):
    match = ROWS_RE.search(plain(description))
    if not match:
        return None
    return float(match.group(1)) * SCALE[match.group(2).lower()]


def shares(breaches, key):
    out = {}
    for year in YEARS:
        rows = [b for b in breaches if b[key][:4] == year]
        total = sum(b["PwnCount"] for b in rows)
        stealer = sum(b["PwnCount"] for b in rows if b.get("IsStealerLog"))
        pw = [b for b in rows if "Passwords" in b["DataClasses"]]
        pw_total = sum(b["PwnCount"] for b in pw)
        pw_stealer = sum(b["PwnCount"] for b in pw if b.get("IsStealerLog"))
        out[year] = {"entries": len(rows), "addresses": total, "stealer": stealer,
                     "share": stealer / total if total else 0.0,
                     "pw_entries": len(pw), "pw_addresses": pw_total,
                     "pw_share": pw_stealer / pw_total if pw_total else 0.0,
                     "largest": max(rows, key=lambda b: b["PwnCount"]) if rows else None}
    return out


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--csv", metavar="FILE", help="write the stealer-log entries as CSV")
    args = parser.parse_args()

    breaches = fetch()
    total = sum(b["PwnCount"] for b in breaches)
    logs = sorted((b for b in breaches if b.get("IsStealerLog")), key=lambda b: b["AddedDate"])
    malware = [b for b in breaches if b.get("IsMalware")]
    print(f"catalogue: {len(breaches):,} entries, {total:,} loaded addresses")
    print(f"flagged IsStealerLog: {len(logs)}   flagged IsMalware: {len(malware)}   "
          f"both: {sum(1 for b in logs if b.get('IsMalware'))}")
    print()

    print("THE STEALER-LOG ENTRIES")
    print(f"  {'name':<32} {'dated':<11} {'loaded':<11} {'unique addresses':>17} {'stated rows':>15} {'rows/address':>13}")
    for b in logs:
        rows = stated_rows(b["Description"])
        ratio = f"{rows / b['PwnCount']:.1f}" if rows else "-"
        print(f"  {b['Name']:<32} {b['BreachDate']:<11} {b['AddedDate'][:10]:<11} {b['PwnCount']:>17,} "
              f"{(f'{rows:,.0f}' if rows else '-'):>15} {ratio:>13}")
    sl_total = sum(b["PwnCount"] for b in logs)
    print(f"  {'all six':<32} {'':<11} {'':<11} {sl_total:>17,}   "
          f"= {100 * sl_total / total:.2f}% of every address ever loaded, from "
          f"{100 * len(logs) / len(breaches):.2f}% of the entries")
    print(f"  data classes: {sorted({c for b in logs for c in b['DataClasses']})}")
    print()

    for key, label in (("AddedDate", "BY YEAR LOADED INTO HIBP"), ("BreachDate", "BY YEAR THE DATA IS DATED")):
        print(label)
        print(f"  {'year':<6} {'entries':>8} {'addresses':>15} {'stealer share':>14} "
              f"{'pw entries':>11} {'stealer share of pw':>20}   largest entry")
        for year, s in shares(breaches, key).items():
            big = s["largest"]
            print(f"  {year:<6} {s['entries']:>8} {s['addresses']:>15,} {100 * s['share']:>13.1f}% "
                  f"{s['pw_entries']:>11} {100 * s['pw_share']:>19.1f}%   "
                  f"{big['Name']} {big['PwnCount']:,}{' (stealer log)' if big.get('IsStealerLog') else ''}")
        print()

    recent = [b for b in breaches if b["AddedDate"][:4] in ("2025", "2026")]
    r_total = sum(b["PwnCount"] for b in recent)
    r_stealer = sum(b["PwnCount"] for b in recent if b.get("IsStealerLog"))
    print(f"LOADED IN 2025 AND 2026 TOGETHER: {len(recent)} entries, {r_total:,} addresses, "
          f"stealer logs {r_stealer:,} = {100 * r_stealer / r_total:.1f}%")

    if args.csv:
        with open(args.csv, "w", encoding="utf-8", newline="") as handle:
            writer = csv.writer(handle)
            writer.writerow(["name", "title", "breach_date", "added_date", "unique_addresses",
                             "stated_rows", "rows_per_address", "data_classes", "is_verified",
                             "is_malware", "hibp_url"])
            for b in logs:
                rows = stated_rows(b["Description"])
                writer.writerow([b["Name"], b["Title"], b["BreachDate"], b["AddedDate"][:10],
                                 b["PwnCount"], f"{rows:.0f}" if rows else "",
                                 f"{rows / b['PwnCount']:.2f}" if rows else "",
                                 "; ".join(b["DataClasses"]), b["IsVerified"], b.get("IsMalware"),
                                 f"https://haveibeenpwned.com/Breach/{b['Name']}"])
        print(f"\nwrote {args.csv}")


if __name__ == "__main__":
    main()